From 6972ecdc7dafaa99ffdfc0aa336479d1f621328e Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 5 Dec 2024 11:41:46 +0900 Subject: [PATCH] GH-44829: [Java][CI] Remove Java related test CI except integration test (#44946) ### Rationale for this change We don't need CI in apache/arrow except integration test because we're moving the Java implementation to apache/arrow-java. ### What changes are included in this PR? Remove Java related CI. ### Are these changes tested? Yes. ### Are there any user-facing changes? No. * GitHub Issue: #44829 Authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- .github/workflows/dev_pr/labeler.yml | 6 - .github/workflows/java.yml | 173 -------------- .github/workflows/java_jni.yml | 147 ------------ .github/workflows/java_nightly.yml | 139 ------------ ci/scripts/install_substrait_consumer.sh | 6 +- ci/scripts/integration_spark.sh | 31 +-- dev/tasks/java-jars/README.md | 29 --- dev/tasks/java-jars/github.yml | 278 ----------------------- dev/tasks/tasks.yml | 180 +-------------- docker-compose.yml | 123 +--------- 10 files changed, 8 insertions(+), 1104 deletions(-) delete mode 100644 .github/workflows/java.yml delete mode 100644 .github/workflows/java_jni.yml delete mode 100644 .github/workflows/java_nightly.yml delete mode 100644 dev/tasks/java-jars/README.md delete mode 100644 dev/tasks/java-jars/github.yml diff --git a/.github/workflows/dev_pr/labeler.yml b/.github/workflows/dev_pr/labeler.yml index fed4b77295bd5..7ef92f0be9b87 100644 --- a/.github/workflows/dev_pr/labeler.yml +++ b/.github/workflows/dev_pr/labeler.yml @@ -35,11 +35,6 @@ - any-glob-to-any-file: - go/**/* -"Component: Java": -- changed-files: - - any-glob-to-any-file: - - java/**/* - "Component: JavaScript": - changed-files: - any-glob-to-any-file: @@ -82,7 +77,6 @@ - any-glob-to-any-file: - c_glib/gandiva-glib/**/* - cpp/src/gandiva/**/* - - java/gandiva/**/* - python/pyarrow/gandiva.* - ruby/red-gandiva/**/* diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml deleted file mode 100644 index f6b3fa748e6d2..0000000000000 --- a/.github/workflows/java.yml +++ /dev/null @@ -1,173 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Java - -on: - push: - branches: - - '**' - - '!dependabot/**' - tags: - - '**' - paths: - - '.dockerignore' - - '.github/workflows/java.yml' - - 'ci/docker/*java*' - - 'ci/scripts/java*.sh' - - 'ci/scripts/util_*.sh' - - 'docker-compose.yml' - - 'format/Flight.proto' - - 'java/**' - pull_request: - paths: - - '.dockerignore' - - '.github/workflows/java.yml' - - 'ci/docker/*java*' - - 'ci/scripts/java*.sh' - - 'ci/scripts/util_*.sh' - - 'docker-compose.yml' - - 'format/Flight.proto' - - 'java/**' - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -env: - ARCHERY_DEBUG: 1 - DOCKER_VOLUME_PREFIX: ".docker/" - -jobs: - ubuntu: - name: AMD64 Ubuntu 22.04 Java JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }} - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - jdk: [11, 17, 21, 22] - maven: [3.9.6] - image: [java] - env: - JDK: ${{ matrix.jdk }} - MAVEN: ${{ matrix.maven }} - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 0 - submodules: recursive - - name: Cache Docker Volumes - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 - with: - path: .docker - key: maven-${{ hashFiles('java/**') }} - restore-keys: maven- - - name: Setup Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 - with: - python-version: 3.12 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} - run: | - archery docker run \ - -e CI=true \ - -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \ - ${{ matrix.image }} - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push ${{ matrix.image }} - - macos: - name: AMD64 macOS 13 Java JDK ${{ matrix.jdk }} - runs-on: macos-13 - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - jdk: [11] - steps: - - name: Set up Java - uses: actions/setup-java@v4 - with: - distribution: 'zulu' - java-version: ${{ matrix.jdk }} - - name: Checkout Arrow - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: recursive - - name: Build - shell: bash - env: - DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} - run: ci/scripts/java_build.sh $(pwd) $(pwd)/build - - name: Test - shell: bash - env: - DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} - run: ci/scripts/java_test.sh $(pwd) $(pwd)/build - - windows: - name: AMD64 Windows Server 2022 Java JDK ${{ matrix.jdk }} - runs-on: windows-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - jdk: [11] - steps: - - name: Set up Java - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.jdk }} - distribution: 'temurin' - - name: Checkout Arrow - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: recursive - - name: Build - shell: bash - env: - DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} - run: ci/scripts/java_build.sh $(pwd) $(pwd)/build - - name: Test - shell: bash - env: - DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} - run: ci/scripts/java_test.sh $(pwd) $(pwd)/build diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml deleted file mode 100644 index 5682f8e84167e..0000000000000 --- a/.github/workflows/java_jni.yml +++ /dev/null @@ -1,147 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Java JNI - -on: - push: - branches: - - '**' - - '!dependabot/**' - tags: - - '**' - paths: - - '.dockerignore' - - '.github/workflows/java_jni.yml' - - 'ci/docker/**' - - 'ci/scripts/cpp_build.sh' - - 'ci/scripts/java_*' - - 'cpp/**' - - 'docker-compose.yml' - - 'java/**' - pull_request: - paths: - - '.dockerignore' - - '.github/workflows/java_jni.yml' - - 'ci/docker/**' - - 'ci/scripts/cpp_build.sh' - - 'ci/scripts/java_*' - - 'cpp/**' - - 'docker-compose.yml' - - 'java/**' - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -permissions: - contents: read - -env: - ARCHERY_DEBUG: 1 - DOCKER_VOLUME_PREFIX: ".docker/" - -jobs: - docker: - name: AMD64 manylinux2014 Java JNI - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 240 - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 0 - submodules: recursive - - name: Free up disk space - run: | - ci/scripts/util_free_space.sh - - name: Cache Docker Volumes - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 - with: - path: .docker - key: java-jni-manylinux-2014-${{ hashFiles('cpp/**', 'java/**') }} - restore-keys: java-jni-manylinux-2014- - - name: Setup Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 - with: - python-version: 3.12 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - run: | - source ci/scripts/util_enable_core_dumps.sh - archery docker run java-jni-manylinux-2014 - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push java-jni-manylinux-2014 - - docker_integration_python: - name: AMD64 Conda Java C Data Interface Integration - runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 90 - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 0 - submodules: recursive - - name: Cache Docker Volumes - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 - with: - path: .docker - key: maven-${{ hashFiles('java/**') }} - restore-keys: maven- - - name: Setup Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 - with: - python-version: 3.12 - - name: Setup Archery - run: pip install -e dev/archery[docker] - - name: Execute Docker Build - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} - run: | - archery docker run \ - -e CI=true \ - -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \ - conda-python-java-integration - - name: Docker Push - if: >- - success() && - github.event_name == 'push' && - github.repository == 'apache/arrow' && - github.ref_name == 'main' - env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - continue-on-error: true - run: archery docker push conda-python-java-integration diff --git a/.github/workflows/java_nightly.yml b/.github/workflows/java_nightly.yml deleted file mode 100644 index 436cc324ddc45..0000000000000 --- a/.github/workflows/java_nightly.yml +++ /dev/null @@ -1,139 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Upload Java Nightly builds - -on: - workflow_dispatch: - inputs: - prefix: - description: Job prefix to use. - required: false - default: '' - keep: - description: Number of versions to keep. - required: false - default: 14 - schedule: - - cron: '0 14 * * *' - -permissions: - contents: read - -jobs: - upload: - if: github.repository == 'apache/arrow' - env: - PREFIX: ${{ github.event.inputs.prefix || ''}} - CROSSBOW_GITHUB_TOKEN: ${{ github.token }} - runs-on: ubuntu-latest - steps: - - name: Checkout Arrow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 1 - path: arrow - repository: apache/arrow - ref: main - submodules: recursive - - name: Checkout Crossbow - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - fetch-depth: 0 - path: crossbow - repository: ursacomputing/crossbow - ref: main - - name: Set up Python - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 - with: - cache: 'pip' - python-version: 3.12 - - name: Install Archery - shell: bash - run: pip install -e arrow/dev/archery[all] - - run: mkdir -p binaries - - name: Download Artifacts - run: | - if [ -z $PREFIX ]; then - PREFIX=nightly-packaging-$(date +%Y-%m-%d)-0 - fi - echo $PREFIX - archery crossbow download-artifacts -f java-jars -t binaries $PREFIX - - name: Sync from Remote - uses: ./arrow/.github/actions/sync-nightlies - with: - switches: -avzh --update --delete --progress - local_path: repo - remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/java - remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }} - remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }} - remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }} - remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }} - remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }} - - shell: bash - name: Show local repo sync from remote - run: | - for i in `ls -t repo/org/apache/arrow`; do - echo "- $i: $(find repo/org/apache/arrow/$i -mindepth 1 -maxdepth 1 -type d \ - | wc -l \ - | xargs) versions available" - done - - shell: bash - name: Build Repository - run: | - DATE=$(date +%Y-%m-%d) - if [ -z $PREFIX ]; then - PREFIX=nightly-packaging-${DATE}-0 - fi - PATTERN_TO_GET_LIB_AND_VERSION='([a-z].+)-([0-9]+.[0-9]+.[0-9]+-SNAPSHOT)' - mkdir -p repo/org/apache/arrow/ - for LIBRARY in $(ls binaries/$PREFIX/java-jars | grep -E '.jar|.json|.pom|.xml' | grep SNAPSHOT); do - [[ $LIBRARY =~ $PATTERN_TO_GET_LIB_AND_VERSION ]] - mkdir -p repo/org/apache/arrow/${BASH_REMATCH[1]}/${BASH_REMATCH[2]} - mkdir -p repo/org/apache/arrow/${BASH_REMATCH[1]}/${DATE} - # Copy twice to maintain a latest snapshot and some earlier versions - cp binaries/$PREFIX/java-jars/$LIBRARY repo/org/apache/arrow/${BASH_REMATCH[1]}/${BASH_REMATCH[2]} - touch repo/org/apache/arrow/${BASH_REMATCH[1]}/${BASH_REMATCH[2]} - cp binaries/$PREFIX/java-jars/$LIBRARY repo/org/apache/arrow/${BASH_REMATCH[1]}/${DATE} - echo "Artifacts $LIBRARY configured" - done - - name: Prune Repository - shell: bash - env: - KEEP: ${{ github.event.inputs.keep || 14 }} - run: | - for i in `ls -t repo/org/apache/arrow`; do - find repo/org/apache/arrow/$i -mindepth 1 -maxdepth 1 -type d -print0 \ - | xargs -0 ls -t -d \ - | tail -n +$((KEEP + 1)) \ - | xargs rm -rf - done - - name: Show repo contents - run: tree repo - - name: Sync to Remote - if: ${{ github.repository == 'apache/arrow' }} - uses: ./arrow/.github/actions/sync-nightlies - with: - upload: true - switches: -avzh --update --delete --progress - local_path: repo - remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/arrow/java - remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }} - remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }} - remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }} - remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }} - remote_host_key: ${{ secrets.NIGHTLIES_RSYNC_HOST_KEY }} diff --git a/ci/scripts/install_substrait_consumer.sh b/ci/scripts/install_substrait_consumer.sh index 8d3333a18f8de..2e6d299f68bf2 100755 --- a/ci/scripts/install_substrait_consumer.sh +++ b/ci/scripts/install_substrait_consumer.sh @@ -24,10 +24,8 @@ echo "Install Substrait Consumer Test Suite"; git clone https://github.com/substrait-io/consumer-testing.git cd consumer-testing # avoid installing pyarrow -grep -v 'pyarrow\|arrow-nightlies' requirements.txt | while read line -do - pip install $line -done +grep -v 'pyarrow\|arrow-nightlies' requirements.txt > requirements-no-arrow.txt +pip install -r requirements-no-arrow.txt pip install -r requirements-build.txt # setup substrait-java diff --git a/ci/scripts/integration_spark.sh b/ci/scripts/integration_spark.sh index 424ac5994653a..4272e2e776d6e 100755 --- a/ci/scripts/integration_spark.sh +++ b/ci/scripts/integration_spark.sh @@ -21,9 +21,6 @@ set -eu source_dir=${1} spark_dir=${2} -# Test Spark with latest PyArrow only, don't build with latest Arrow Java -test_pyarrow_only=${3:-false} - # Spark branch to checkout spark_version=${SPARK_VERSION:-master} @@ -44,32 +41,10 @@ export MAVEN_OPTS="-Xss256m -Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simp export MAVEN_OPTS="${MAVEN_OPTS} -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" pushd ${spark_dir} + echo "Building Spark ${SPARK_VERSION}" - if [ "${test_pyarrow_only}" == "true" ]; then - echo "Building Spark ${SPARK_VERSION} to test pyarrow only" - - # Build Spark only - build/mvn -B -DskipTests package - - else - - # Update Spark pom with the Arrow version just installed and build Spark, need package phase for pyspark - echo "Building Spark ${SPARK_VERSION} with Arrow ${arrow_version}" - build/mvn versions:set-property -Dproperty=arrow.version -DnewVersion=${arrow_version} - - # Build Spark with new Arrow Java - build/mvn -B -DskipTests package - - spark_scala_tests=( - "org.apache.spark.sql.execution.arrow" - "org.apache.spark.sql.execution.vectorized.ColumnarBatchSuite" - "org.apache.spark.sql.execution.vectorized.ArrowColumnVectorSuite") - - (echo "Testing Spark:"; IFS=$'\n'; echo "${spark_scala_tests[*]}") - - # TODO: should be able to only build spark-sql tests with adding "-pl sql/core" but not currently working - build/mvn -B -Dtest=none -DwildcardSuites=$(IFS=,; echo "${spark_scala_tests[*]}") test - fi + # Build Spark only + build/mvn -B -DskipTests package # Run pyarrow related Python tests only spark_python_tests=( diff --git a/dev/tasks/java-jars/README.md b/dev/tasks/java-jars/README.md deleted file mode 100644 index 216c7198d3239..0000000000000 --- a/dev/tasks/java-jars/README.md +++ /dev/null @@ -1,29 +0,0 @@ - - -# Java Jars Task - -This directory is responsible to generate the jar files for the Arrow components that depend on C++ shared libraries to execute. - -The Arrow C++ libraries are compiled both on macOS and Linux distributions, with their dependencies linked statically, and they are added -in the jars at the end, so the file can be used on both systems. - -## Linux Docker Image -To compile the C++ libraries in Linux, a docker image is used. -It is created used the **ci/docker/java-bundled-jars.dockerfile** file. -If it is necessary to add any new dependency, you need to change that file. \ No newline at end of file diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml deleted file mode 100644 index affa26a1d9332..0000000000000 --- a/dev/tasks/java-jars/github.yml +++ /dev/null @@ -1,278 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -{% import 'macros.jinja' as macros with context %} - -{{ macros.github_header() }} - -permissions: - packages: write - -jobs: - - build-cpp-ubuntu: - {% set arch = '${{ matrix.platform.arch }}' %} - name: Build C++ libraries Ubuntu {{ arch }} - runs-on: {{ '${{ matrix.platform.runs_on }}' }} - env: - # architecture name used for archery build - ARCH: {{ '${{ matrix.platform.archery_arch }}' }} - ARCH_ALIAS: {{ '${{ matrix.platform.archery_arch_alias }}' }} - ARCH_SHORT: {{ '${{ matrix.platform.archery_arch_short }}' }} - strategy: - fail-fast: false - matrix: - platform: - - runs_on: ["ubuntu-latest"] - arch: "x86_64" - archery_arch: "amd64" - archery_arch_alias: "x86_64" - archery_arch_short: "amd64" - - runs_on: ["self-hosted", "Linux", "arm64"] - arch: "aarch_64" - archery_arch: "arm64v8" - archery_arch_alias: "aarch64" - archery_arch_short: "arm64" - steps: - {{ macros.github_checkout_arrow()|indent }} - {{ macros.github_free_space()|indent }} - {{ macros.github_install_archery()|indent }} - - name: Build C++ libraries - env: - {{ macros.github_set_sccache_envvars()|indent(8) }} - GITHUB_TOKEN: {{ '${{ secrets.GITHUB_TOKEN }}' }} - run: | - if [ "${ARCH}" = "arm64v8" ]; then - # We can't use NuGet on manylinux2014_aarch64 because Mono is old. - : - else - export VCPKG_BINARY_SOURCES="clear;nuget,GitHub,readwrite" - fi - archery docker run \ - -e ARROW_JAVA_BUILD=OFF \ - -e ARROW_JAVA_TEST=OFF \ - java-jni-manylinux-2014 - - name: Compress into single artifact to keep directory structure - run: tar -cvzf arrow-shared-libs-linux-{{ arch }}.tar.gz arrow/java-dist/ - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: ubuntu-shared-lib-{{ arch }} - path: arrow-shared-libs-linux-{{ arch }}.tar.gz - {% if arrow.is_default_branch() %} - {{ macros.github_login_dockerhub()|indent }} - - name: Push Docker image - shell: bash - run: archery docker push java-jni-manylinux-2014 - {% endif %} - - build-cpp-macos: - {% set arch = '${{ matrix.platform.arch }}' %} - name: Build C++ libraries macOS {{ arch }} - runs-on: {{ '${{ matrix.platform.runs_on }}' }} - strategy: - fail-fast: false - matrix: - platform: - - { runs_on: ["macos-13"], arch: "x86_64"} - - { runs_on: ["macos-14"], arch: "aarch_64" } - env: - MACOSX_DEPLOYMENT_TARGET: "14.0" - steps: - {{ macros.github_checkout_arrow()|indent }} - - name: Set up Python - uses: actions/setup-python@v4 - with: - cache: 'pip' - python-version: 3.12 - - name: Install Archery - shell: bash - run: pip install -e arrow/dev/archery[all] - - name: Install dependencies - run: | - # We want to use llvm@14 to avoid shared z3 - # dependency. llvm@14 doesn't depend on z3 and llvm depends - # on z3. And Homebrew's z3 provides only shared library. It - # doesn't provides static z3 because z3's CMake doesn't accept - # building both shared and static libraries at once. - # See also: Z3_BUILD_LIBZ3_SHARED in - # https://github.com/Z3Prover/z3/blob/master/README-CMake.md - # - # If llvm is installed, Apache Arrow C++ uses llvm rather than - # llvm@14 because llvm is newer than llvm@14. - brew uninstall llvm || : - - # Ensure updating python@XXX with the "--overwrite" option. - # If python@XXX is updated without "--overwrite", it causes - # a conflict error. Because Python 3 installed not by - # Homebrew exists in /usr/local on GitHub Actions. If - # Homebrew's python@XXX is updated without "--overwrite", it - # tries to replace /usr/local/bin/2to3 and so on and causes - # a conflict error. - brew update - for python_package in $(brew list | grep python@); do - brew install --overwrite ${python_package} - done - brew install --overwrite python - - if [ "$(uname -m)" = "arm64" ]; then - # pkg-config formula is deprecated but it's still installed - # in GitHub Actions runner now. We can remove this once - # pkg-config formula is removed from GitHub Actions runner. - brew uninstall pkg-config || : - brew uninstall pkg-config@0.29.2 || : - fi - - brew bundle --file=arrow/cpp/Brewfile - # We want to link aws-sdk-cpp statically but Homebrew's - # aws-sdk-cpp provides only shared library. If we have - # Homebrew's aws-sdk-cpp, our build mix Homebrew's - # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's - # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp. - brew uninstall aws-sdk-cpp - # We want to use bundled RE2 for static linking. If - # Homebrew's RE2 is installed, its header file may be used. - # We uninstall Homebrew's RE2 to ensure using bundled RE2. - brew uninstall grpc || : # gRPC depends on RE2 - brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too - brew uninstall re2 - # We want to use bundled Protobuf for static linking. If - # Homebrew's Protobuf is installed, its library file may be - # used on test We uninstall Homebrew's Protobuf to ensure using - # bundled Protobuf. - brew uninstall protobuf - - brew bundle --file=arrow/java/Brewfile - - name: Build C++ libraries - env: - {{ macros.github_set_sccache_envvars()|indent(8) }} - run: | - set -e - # make brew Java available to CMake - export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home - arrow/ci/scripts/java_jni_macos_build.sh \ - $GITHUB_WORKSPACE/arrow \ - $GITHUB_WORKSPACE/arrow/cpp-build \ - $GITHUB_WORKSPACE/arrow/java-dist - - name: Compress into single artifact to keep directory structure - run: tar -cvzf arrow-shared-libs-macos-{{ arch }}.tar.gz arrow/java-dist/ - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: macos-shared-lib-{{ arch }} - path: arrow-shared-libs-macos-{{ arch }}.tar.gz - - build-cpp-windows: - name: Build C++ libraries Windows - runs-on: windows-2019 - steps: - {{ macros.github_checkout_arrow()|indent }} - - name: Set up Java - uses: actions/setup-java@v3 - with: - java-version: '11' - distribution: 'temurin' - - name: Download Timezone Database - shell: bash - run: arrow/ci/scripts/download_tz_database.sh - - name: Install sccache - shell: bash - run: arrow/ci/scripts/install_sccache.sh pc-windows-msvc $(pwd)/sccache - - name: Build C++ libraries - shell: cmd - env: - {{ macros.github_set_sccache_envvars()|indent(8) }} - run: | - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - REM For ORC - set TZDIR=/c/msys64/usr/share/zoneinfo - bash -c "arrow/ci/scripts/java_jni_windows_build.sh $(pwd)/arrow $(pwd)/arrow/cpp-build $(pwd)/arrow/java-dist" - - name: Compress into single artifact to keep directory structure - shell: bash - run: tar -cvzf arrow-shared-libs-windows.tar.gz arrow/java-dist/ - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: windows-shared-lib - path: arrow-shared-libs-windows.tar.gz - - package-jars: - name: Build jar files - runs-on: macos-latest - needs: - - build-cpp-ubuntu - - build-cpp-macos - - build-cpp-windows - steps: - {{ macros.github_checkout_arrow(fetch_depth=0)|indent }} - - name: Download Libraries - uses: actions/download-artifact@v4 - with: - path: artifacts - - name: Decompress artifacts - run: | - mv artifacts/*/*.tar.gz . - tar -xvzf arrow-shared-libs-linux-x86_64.tar.gz - tar -xvzf arrow-shared-libs-linux-aarch_64.tar.gz - tar -xvzf arrow-shared-libs-macos-x86_64.tar.gz - tar -xvzf arrow-shared-libs-macos-aarch_64.tar.gz - tar -xvzf arrow-shared-libs-windows.tar.gz - - name: Test that shared libraries exist - run: | - set -x - - test -f arrow/java-dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.so - test -f arrow/java-dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.so - test -f arrow/java-dist/arrow_orc_jni/x86_64/libarrow_orc_jni.so - test -f arrow/java-dist/gandiva_jni/x86_64/libgandiva_jni.so - - test -f arrow/java-dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.so - test -f arrow/java-dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.so - test -f arrow/java-dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.so - test -f arrow/java-dist/gandiva_jni/aarch_64/libgandiva_jni.so - - test -f arrow/java-dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.dylib - test -f arrow/java-dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.dylib - test -f arrow/java-dist/arrow_orc_jni/x86_64/libarrow_orc_jni.dylib - test -f arrow/java-dist/gandiva_jni/x86_64/libgandiva_jni.dylib - - test -f arrow/java-dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.dylib - test -f arrow/java-dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.dylib - test -f arrow/java-dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib - test -f arrow/java-dist/gandiva_jni/aarch_64/libgandiva_jni.dylib - - test -f arrow/java-dist/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll - test -f arrow/java-dist/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll - test -f arrow/java-dist/arrow_orc_jni/x86_64/arrow_orc_jni.dll - - name: Build bundled jar - env: - MAVEN_ARGS: >- - --no-transfer-progress - run: | - set -e - pushd arrow/java - mvn versions:set -DnewVersion={{ arrow.no_rc_snapshot_version }} - mvn versions:set -DnewVersion={{ arrow.no_rc_snapshot_version }} -f bom - popd - arrow/ci/scripts/java_full_build.sh \ - $GITHUB_WORKSPACE/arrow \ - $GITHUB_WORKSPACE/arrow/java-dist - {{ macros.github_upload_releases(["arrow/java-dist/*.jar", - "arrow/java-dist/*.json", - "arrow/java-dist/*.pom", - "arrow/java-dist/*.xml", - "arrow/java-dist/*.zip"])|indent }} diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 69235bf0cf4fe..c8c311f5137df 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -59,7 +59,6 @@ groups: - centos-* - conan-* - debian-* - - java-jars - matlab - nuget - python-sdist @@ -83,10 +82,6 @@ groups: c-glib: - test-*c-glib* - java: - - "*java*" - - test-*spark* - python: - test-*python* - example-*python* @@ -102,7 +97,6 @@ groups: vcpkg: - test-*vcpkg* - wheel-* - - java-jars integration: - test-*dask* @@ -154,7 +148,6 @@ groups: - centos-* - conan-* - conda-* - - java-jars - homebrew-cpp - nuget - test-* @@ -184,7 +177,6 @@ groups: - ~conda-linux-x64-cuda-py3 - ~conda-osx-arm64-cpu-py3 - conan-* - - java-jars - homebrew-cpp - nuget - wheel-* @@ -695,173 +687,6 @@ tasks: artifacts: - matlab-arrow-{no_rc_no_dev_version}.mltbx - ############################## Arrow JAR's ################################## - - java-jars: - # Build jar's that contains cpp libraries dependencies - ci: github - template: java-jars/github.yml - artifacts: - - arrow-algorithm-{no_rc_snapshot_version}-cyclonedx.json - - arrow-algorithm-{no_rc_snapshot_version}-cyclonedx.xml - - arrow-algorithm-{no_rc_snapshot_version}-javadoc.jar - - arrow-algorithm-{no_rc_snapshot_version}-sources.jar - - arrow-algorithm-{no_rc_snapshot_version}-tests.jar - - arrow-algorithm-{no_rc_snapshot_version}.jar - - arrow-algorithm-{no_rc_snapshot_version}.pom - - arrow-avro-{no_rc_snapshot_version}-cyclonedx.json - - arrow-avro-{no_rc_snapshot_version}-cyclonedx.xml - - arrow-avro-{no_rc_snapshot_version}-javadoc.jar - - arrow-avro-{no_rc_snapshot_version}-sources.jar - - arrow-avro-{no_rc_snapshot_version}-tests.jar - - arrow-avro-{no_rc_snapshot_version}.jar - - arrow-avro-{no_rc_snapshot_version}.pom - - arrow-bom-{no_rc_snapshot_version}.pom - - arrow-c-data-{no_rc_snapshot_version}-cyclonedx.json - - arrow-c-data-{no_rc_snapshot_version}-cyclonedx.xml - - arrow-c-data-{no_rc_snapshot_version}-javadoc.jar - - arrow-c-data-{no_rc_snapshot_version}-sources.jar - - arrow-c-data-{no_rc_snapshot_version}-tests.jar - - arrow-c-data-{no_rc_snapshot_version}.jar - - arrow-c-data-{no_rc_snapshot_version}.pom - - arrow-compression-{no_rc_snapshot_version}-cyclonedx.json - - arrow-compression-{no_rc_snapshot_version}-cyclonedx.xml - - arrow-compression-{no_rc_snapshot_version}-javadoc.jar - - arrow-compression-{no_rc_snapshot_version}-sources.jar - - arrow-compression-{no_rc_snapshot_version}-tests.jar - - arrow-compression-{no_rc_snapshot_version}.jar - - arrow-compression-{no_rc_snapshot_version}.pom - - arrow-dataset-{no_rc_snapshot_version}-cyclonedx.json - - arrow-dataset-{no_rc_snapshot_version}-cyclonedx.xml - - arrow-dataset-{no_rc_snapshot_version}-javadoc.jar - - arrow-dataset-{no_rc_snapshot_version}-sources.jar - - arrow-dataset-{no_rc_snapshot_version}-tests.jar - - arrow-dataset-{no_rc_snapshot_version}.jar - - arrow-dataset-{no_rc_snapshot_version}.pom - - arrow-flight-{no_rc_snapshot_version}-cyclonedx.json - - arrow-flight-{no_rc_snapshot_version}-cyclonedx.xml - - arrow-flight-{no_rc_snapshot_version}.pom - - arrow-format-{no_rc_snapshot_version}-cyclonedx.json - - arrow-format-{no_rc_snapshot_version}-cyclonedx.xml - - arrow-format-{no_rc_snapshot_version}-javadoc.jar - - arrow-format-{no_rc_snapshot_version}-sources.jar - - arrow-format-{no_rc_snapshot_version}-tests.jar - - arrow-format-{no_rc_snapshot_version}.jar - - arrow-format-{no_rc_snapshot_version}.pom - - arrow-gandiva-{no_rc_snapshot_version}-cyclonedx.json - - arrow-gandiva-{no_rc_snapshot_version}-cyclonedx.xml - - arrow-gandiva-{no_rc_snapshot_version}-javadoc.jar - - arrow-gandiva-{no_rc_snapshot_version}-sources.jar - - arrow-gandiva-{no_rc_snapshot_version}-tests.jar - - arrow-gandiva-{no_rc_snapshot_version}.jar - - arrow-gandiva-{no_rc_snapshot_version}.pom - - arrow-java-root-{no_rc_snapshot_version}-cyclonedx.json - - arrow-java-root-{no_rc_snapshot_version}-cyclonedx.xml - - arrow-java-root-{no_rc_snapshot_version}-source-release.zip - - arrow-java-root-{no_rc_snapshot_version}.pom - - arrow-jdbc-{no_rc_snapshot_version}-cyclonedx.json - - arrow-jdbc-{no_rc_snapshot_version}-cyclonedx.xml - - arrow-jdbc-{no_rc_snapshot_version}-javadoc.jar - - arrow-jdbc-{no_rc_snapshot_version}-sources.jar - - arrow-jdbc-{no_rc_snapshot_version}-tests.jar - - arrow-jdbc-{no_rc_snapshot_version}.jar - - arrow-jdbc-{no_rc_snapshot_version}.pom - - arrow-memory-core-{no_rc_snapshot_version}-cyclonedx.json - - arrow-memory-core-{no_rc_snapshot_version}-cyclonedx.xml - - arrow-memory-core-{no_rc_snapshot_version}-javadoc.jar - - arrow-memory-core-{no_rc_snapshot_version}-sources.jar - - arrow-memory-core-{no_rc_snapshot_version}-tests.jar - - arrow-memory-core-{no_rc_snapshot_version}.jar - - arrow-memory-core-{no_rc_snapshot_version}.pom - - arrow-memory-netty-{no_rc_snapshot_version}-cyclonedx.json - - arrow-memory-netty-{no_rc_snapshot_version}-cyclonedx.xml - - arrow-memory-netty-{no_rc_snapshot_version}-javadoc.jar - - arrow-memory-netty-{no_rc_snapshot_version}-sources.jar - - arrow-memory-netty-{no_rc_snapshot_version}-tests.jar - - arrow-memory-netty-{no_rc_snapshot_version}.jar - - arrow-memory-netty-{no_rc_snapshot_version}.pom - - arrow-memory-netty-buffer-patch-{no_rc_snapshot_version}-cyclonedx.json - - arrow-memory-netty-buffer-patch-{no_rc_snapshot_version}-cyclonedx.xml - - arrow-memory-netty-buffer-patch-{no_rc_snapshot_version}-javadoc.jar - - arrow-memory-netty-buffer-patch-{no_rc_snapshot_version}-sources.jar - - arrow-memory-netty-buffer-patch-{no_rc_snapshot_version}-tests.jar - - arrow-memory-netty-buffer-patch-{no_rc_snapshot_version}.jar - - arrow-memory-netty-buffer-patch-{no_rc_snapshot_version}.pom - - arrow-memory-unsafe-{no_rc_snapshot_version}-cyclonedx.json - - arrow-memory-unsafe-{no_rc_snapshot_version}-cyclonedx.xml - - arrow-memory-unsafe-{no_rc_snapshot_version}-javadoc.jar - - arrow-memory-unsafe-{no_rc_snapshot_version}-sources.jar - - arrow-memory-unsafe-{no_rc_snapshot_version}-tests.jar - - arrow-memory-unsafe-{no_rc_snapshot_version}.jar - - arrow-memory-unsafe-{no_rc_snapshot_version}.pom - - arrow-memory-{no_rc_snapshot_version}-cyclonedx.json - - arrow-memory-{no_rc_snapshot_version}-cyclonedx.xml - - arrow-memory-{no_rc_snapshot_version}.pom - - arrow-orc-{no_rc_snapshot_version}-cyclonedx.json - - arrow-orc-{no_rc_snapshot_version}-cyclonedx.xml - - arrow-orc-{no_rc_snapshot_version}-javadoc.jar - - arrow-orc-{no_rc_snapshot_version}-sources.jar - - arrow-orc-{no_rc_snapshot_version}-tests.jar - - arrow-orc-{no_rc_snapshot_version}.jar - - arrow-orc-{no_rc_snapshot_version}.pom - - arrow-performance-{no_rc_snapshot_version}-cyclonedx.json - - arrow-performance-{no_rc_snapshot_version}-cyclonedx.xml - - arrow-performance-{no_rc_snapshot_version}-sources.jar - - arrow-performance-{no_rc_snapshot_version}-tests.jar - - arrow-performance-{no_rc_snapshot_version}.jar - - arrow-performance-{no_rc_snapshot_version}.pom - - arrow-tools-{no_rc_snapshot_version}-cyclonedx.json - - arrow-tools-{no_rc_snapshot_version}-cyclonedx.xml - - arrow-tools-{no_rc_snapshot_version}-jar-with-dependencies.jar - - arrow-tools-{no_rc_snapshot_version}-javadoc.jar - - arrow-tools-{no_rc_snapshot_version}-sources.jar - - arrow-tools-{no_rc_snapshot_version}-tests.jar - - arrow-tools-{no_rc_snapshot_version}.jar - - arrow-tools-{no_rc_snapshot_version}.pom - - arrow-vector-{no_rc_snapshot_version}-cyclonedx.json - - arrow-vector-{no_rc_snapshot_version}-cyclonedx.xml - - arrow-vector-{no_rc_snapshot_version}-javadoc.jar - - arrow-vector-{no_rc_snapshot_version}-shade-format-flatbuffers.jar - - arrow-vector-{no_rc_snapshot_version}-sources.jar - - arrow-vector-{no_rc_snapshot_version}-tests.jar - - arrow-vector-{no_rc_snapshot_version}.jar - - arrow-vector-{no_rc_snapshot_version}.pom - - flight-core-{no_rc_snapshot_version}-cyclonedx.json - - flight-core-{no_rc_snapshot_version}-cyclonedx.xml - - flight-core-{no_rc_snapshot_version}-javadoc.jar - - flight-core-{no_rc_snapshot_version}-sources.jar - - flight-core-{no_rc_snapshot_version}-tests.jar - - flight-core-{no_rc_snapshot_version}.jar - - flight-core-{no_rc_snapshot_version}.pom - - flight-integration-tests-{no_rc_snapshot_version}-cyclonedx.json - - flight-integration-tests-{no_rc_snapshot_version}-cyclonedx.xml - - flight-integration-tests-{no_rc_snapshot_version}-jar-with-dependencies.jar - - flight-integration-tests-{no_rc_snapshot_version}-javadoc.jar - - flight-integration-tests-{no_rc_snapshot_version}-sources.jar - - flight-integration-tests-{no_rc_snapshot_version}-tests.jar - - flight-integration-tests-{no_rc_snapshot_version}.jar - - flight-integration-tests-{no_rc_snapshot_version}.pom - - flight-sql-{no_rc_snapshot_version}-cyclonedx.json - - flight-sql-{no_rc_snapshot_version}-cyclonedx.xml - - flight-sql-{no_rc_snapshot_version}-javadoc.jar - - flight-sql-{no_rc_snapshot_version}-sources.jar - - flight-sql-{no_rc_snapshot_version}-tests.jar - - flight-sql-{no_rc_snapshot_version}.jar - - flight-sql-{no_rc_snapshot_version}.pom - - flight-sql-jdbc-core-{no_rc_snapshot_version}-cyclonedx.json - - flight-sql-jdbc-core-{no_rc_snapshot_version}-cyclonedx.xml - - flight-sql-jdbc-core-{no_rc_snapshot_version}-javadoc.jar - - flight-sql-jdbc-core-{no_rc_snapshot_version}-sources.jar - - flight-sql-jdbc-core-{no_rc_snapshot_version}-tests.jar - - flight-sql-jdbc-core-{no_rc_snapshot_version}.jar - - flight-sql-jdbc-core-{no_rc_snapshot_version}.pom - - flight-sql-jdbc-driver-{no_rc_snapshot_version}-cyclonedx.json - - flight-sql-jdbc-driver-{no_rc_snapshot_version}-cyclonedx.xml - - flight-sql-jdbc-driver-{no_rc_snapshot_version}-sources.jar - - flight-sql-jdbc-driver-{no_rc_snapshot_version}-tests.jar - - flight-sql-jdbc-driver-{no_rc_snapshot_version}.jar - - flight-sql-jdbc-driver-{no_rc_snapshot_version}.pom - ############################## NuGet packages ############################### nuget: @@ -928,7 +753,6 @@ tasks: {% for target in ["cpp", "csharp", "integration", - "java", "js", "python", "ruby"] %} @@ -988,7 +812,6 @@ tasks: {% for target in ["cpp", "csharp", "integration", - "java", "js", "python", "ruby"] %} @@ -1013,7 +836,6 @@ tasks: env: ARROW_FLIGHT: 0 ARROW_GANDIVA: 0 - TEST_INTEGRATION_JAVA: 0 PYTEST_ADDOPTS: "-k 'not test_cancellation'" target: {{ target }} github_runner: "macos-14" @@ -1354,7 +1176,7 @@ tasks: test-r-extra-packages: ci: github - template: r/github.linux.extra.packages.yml + template: r/github.linux.extra.packages.yml test-r-linux-as-cran: ci: github diff --git a/docker-compose.yml b/docker-compose.yml index 6ee1bf4d116c6..c2b6ae23f5d44 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -127,8 +127,6 @@ x-hierarchy: - conda-python-dask - conda-python-emscripten - conda-python-hdfs - - conda-python-java-integration - - conda-python-jpype - conda-python-no-numpy - conda-python-spark - conda-python-substrait @@ -142,7 +140,6 @@ x-hierarchy: - debian-js - fedora-cpp: - fedora-python - - java - python-sdist - ubuntu-cpp: - ubuntu-cpp-static @@ -173,8 +170,7 @@ x-hierarchy: # helper services - impala - postgres - - python-wheel-manylinux-2014: - - java-jni-manylinux-2014 + - python-wheel-manylinux-2014 - python-wheel-manylinux-2-28 - python-wheel-manylinux-test-imports - python-free-threaded-wheel-manylinux-test-imports @@ -1301,28 +1297,6 @@ services: target: "C:/arrow" command: arrow\\ci\\scripts\\python_wheel_windows_test.bat - java-jni-manylinux-2014: - image: ${REPO}:${ARCH}-java-jni-manylinux-2014-vcpkg-${VCPKG} - build: - args: - base: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2014-vcpkg-${VCPKG} - java: 11 - context: . - dockerfile: ci/docker/java-jni-manylinux-201x.dockerfile - cache_from: - - ${REPO}:${ARCH}-java-jni-manylinux-2014-vcpkg-${VCPKG} - environment: - <<: [*common, *ccache] - volumes: - - .:/arrow:delegated - - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated - - ${DOCKER_VOLUME_PREFIX}python-wheel-manylinux2014-ccache:/ccache:delegated - command: - ["pip install -e /arrow/dev/archery && \ - /arrow/ci/scripts/java_jni_manylinux_build.sh /arrow /build /arrow/java-dist && \ - /arrow/ci/scripts/java_build.sh /arrow /build /arrow/java-dist && \ - /arrow/ci/scripts/java_test.sh /arrow /build /arrow/java-dist"] - ############################## Integration ################################# conda-python-pandas: @@ -1478,79 +1452,6 @@ services: /arrow/ci/scripts/python_build.sh /arrow /build && /arrow/ci/scripts/integration_substrait.sh"] - conda-python-jpype: - # Usage: - # docker compose build conda - # docker compose build conda-cpp - # docker compose build conda-python - # docker compose build conda-python-jpype - # docker compose run --rm conda-python-jpype - image: ${REPO}:${ARCH}-conda-python-${PYTHON}-jpype - build: - context: . - dockerfile: ci/docker/conda-python-jpype.dockerfile - cache_from: - - ${REPO}:${ARCH}-conda-python-${PYTHON}-jpype - args: - repo: ${REPO} - arch: ${ARCH} - python: ${PYTHON} - shm_size: *shm-size - environment: - <<: [*common, *ccache] - ARROW_FLIGHT: "OFF" - ARROW_FLIGHT_SQL: "OFF" - ARROW_GANDIVA: "OFF" - volumes: *conda-volumes - command: - ["/arrow/ci/scripts/cpp_build.sh /arrow /build && - /arrow/ci/scripts/python_build.sh /arrow /build && - /arrow/ci/scripts/java_build.sh /arrow /build && - /arrow/ci/scripts/python_test.sh /arrow"] - - conda-python-java-integration: - # Usage: - # docker compose build conda - # docker compose build conda-cpp - # docker compose build conda-python - # docker compose build conda-python-java-integration - # docker compose run --rm conda-python-java-integration - image: ${REPO}:${ARCH}-conda-python-${PYTHON}-java-integration - build: - context: . - dockerfile: ci/docker/conda-python-jpype.dockerfile - cache_from: - - ${REPO}:${ARCH}-conda-python-${PYTHON}-java-integration - args: - repo: ${REPO} - arch: ${ARCH} - python: ${PYTHON} - llvm: ${LLVM} - shm_size: *shm-size - environment: - <<: [*common, *ccache] - ARROW_ACERO: "OFF" - ARROW_DATASET: "OFF" - ARROW_FLIGHT: "OFF" - ARROW_FLIGHT_SQL: "OFF" - ARROW_GANDIVA: "OFF" - ARROW_JAVA_CDATA: "ON" - ARROW_ORC: "OFF" - ARROW_PARQUET: "OFF" - JAVA_JNI_CMAKE_ARGS: >- - -DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF - -DARROW_JAVA_JNI_ENABLE_C=ON - volumes: - - .:/arrow:delegated - - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated - - ${DOCKER_VOLUME_PREFIX}debian-ccache:/ccache:delegated - command: - [ "/arrow/ci/scripts/cpp_build.sh /arrow /build && - /arrow/ci/scripts/python_build.sh /arrow /build && - /arrow/ci/scripts/java_jni_build.sh /arrow $${ARROW_HOME} /build /tmp/dist/java/ && - /arrow/ci/scripts/java_build.sh /arrow /build /tmp/dist/java && - /arrow/ci/scripts/java_cdata_integration.sh /arrow /build" ] - conda-python-cython2: # Usage: # docker compose build conda @@ -1836,25 +1737,6 @@ services: /arrow/ci/scripts/csharp_test.sh /arrow && /arrow/ci/scripts/csharp_pack.sh /arrow" - ################################ Java ####################################### - - java: - # Usage: - # docker compose build java - # docker compose run java - # Parameters: - # MAVEN: 3.9.5 - # JDK: 11, 17, 21 - image: ${ARCH}/maven:${MAVEN}-eclipse-temurin-${JDK} - shm_size: *shm-size - volumes: &java-volumes - - .:/arrow:delegated - - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated - command: &java-command > - /bin/bash -c " - /arrow/ci/scripts/java_build.sh /arrow /build && - /arrow/ci/scripts/java_test.sh /arrow /build" - ############################## Integration ################################## conda-integration: @@ -2070,8 +1952,7 @@ services: command: ["/arrow/ci/scripts/cpp_build.sh /arrow /build && /arrow/ci/scripts/python_build.sh /arrow /build && - /arrow/ci/scripts/java_build.sh /arrow /build && - /arrow/ci/scripts/integration_spark.sh /arrow /spark ${TEST_PYARROW_ONLY:-false}"] + /arrow/ci/scripts/integration_spark.sh /arrow /spark"] ################################# Source Verification #####################################