Merge branch 'main' into parquet/support-write-bloom-filter

.env

@@ -89,13 +89,13 @@ TZ=UTC
 # Used through docker-compose.yml and serves as the default version for the
 # ci/scripts/install_vcpkg.sh script. Prefer to use short SHAs to keep the
 # docker tags more readable.
-VCPKG="a42af01b72c28a8e1d7b48107b33e4f286a55ef6"    # 2023.11.20 Release
+VCPKG="943c5ef1c8f6b5e6ced092b242c8299caae2ff01"    # 2024.04.26 Release
 
 # This must be updated when we update
 # ci/docker/python-wheel-windows-vs2019.dockerfile.
 # This is a workaround for our CI problem that "archery docker build" doesn't
 # use pulled built images in dev/tasks/python-wheels/github.windows.yml.
-PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2024-04-09
+PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2024-06-18
 
 # Use conanio/${CONAN_BASE}:{CONAN_VERSION} for "docker-compose run --rm conan".
 # See https://github.com/conan-io/conan-docker-tools#readme and

.github/CODEOWNERS

@@ -37,7 +37,7 @@
 /go/ @zeroshade
 /java/ @lidavidm
 /js/ @domoritz @trxcllnt
-/matlab/ @kevingurney @kou
+/matlab/ @kevingurney @kou @sgilmore10
 /python/pyarrow/_flight.pyx @lidavidm
 /python/pyarrow/**/*gandiva* @wjones127
 /r/ @paleolimbot @thisisnic

.github/workflows/dev.yml

@@ -120,7 +120,7 @@ jobs:
         shell: bash
         run: |
           gem install test-unit
-          pip install "cython>=0.29.31" setuptools six pytest jira setuptools-scm
+          pip install "cython>=0.29.31" setuptools pytest jira setuptools-scm
       - name: Run Release Test
         env:
           ARROW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/go.yml

@@ -306,8 +306,8 @@ jobs:
         run: ci/scripts/go_test.sh $(pwd)
 
   macos:
-    name: AMD64 macOS 11 Go ${{ matrix.go }}
-    runs-on: macos-latest
+    name: AMD64 macOS 12 Go ${{ matrix.go }}
+    runs-on: macos-12
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 60
     strategy:
@@ -364,8 +364,8 @@ jobs:
 
 
   macos-cgo:
-    name: AMD64 macOS 11 Go ${{ matrix.go }} - CGO
-    runs-on: macos-latest
+    name: AMD64 macOS 12 Go ${{ matrix.go }} - CGO
+    runs-on: macos-12
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 60
     strategy:

.github/workflows/java.yml

@@ -105,9 +105,9 @@ jobs:
         run: archery docker push ${{ matrix.image }}
 
   macos:
-    name: AMD64 macOS 11 Java JDK ${{ matrix.jdk }}
-    runs-on: macos-latest
-    if: github.event_name == 'push'
+    name: AMD64 macOS 12 Java JDK ${{ matrix.jdk }}
+    runs-on: macos-12
+    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 30
     strategy:
       fail-fast: false

.github/workflows/java_jni.yml

@@ -54,7 +54,7 @@ jobs:
     name: AMD64 manylinux2014 Java JNI
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
-    timeout-minutes: 90
+    timeout-minutes: 240
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0

.github/workflows/js.yml

@@ -81,10 +81,10 @@ jobs:
         run: archery docker push debian-js
 
   macos:
-    name: AMD64 macOS 11 NodeJS ${{ matrix.node }}
-    runs-on: macos-latest
-    if: github.event_name == 'push'
-    timeout-minutes: 90
+    name: AMD64 macOS 12 NodeJS ${{ matrix.node }}
+    runs-on: macos-12
+    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
+    timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
@@ -114,7 +114,8 @@ jobs:
   windows:
     name: AMD64 Windows NodeJS ${{ matrix.node }}
     runs-on: windows-latest
-    if: github.event_name == 'push'
+    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
+    timeout-minutes: 45
     strategy:
       fail-fast: false
       matrix:

.github/workflows/release.yml

@@ -0,0 +1,74 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Release
+
+on:
+  push:
+    tags:
+      # Trigger workflow when a tag whose name matches the pattern
+      # pattern "apache-arrow-{MAJOR}.{MINOR}.{PATCH}" is pushed.
+      - "apache-arrow-[0-9]+.[0-9]+.[0-9]+"
+
+permissions:
+  contents: write
+
+env:
+  GH_TOKEN: ${{ github.token }}
+
+jobs:
+  publish:
+    name: Publish 
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Get Tag Name of Latest Release Candidate
+        run: |
+          rc_tag=$(gh release list --repo apache/arrow | \
+                    cut -f3 | \
+                    grep -F "${GITHUB_REF_NAME}-rc" | \
+                    head -n1)
+          echo "Latest Release Candidate Tag: ${rc_tag}"
+          echo "RELEASE_CANDIDATE_TAG_NAME=${rc_tag}" >> ${GITHUB_ENV}
+      - name: Store Version and Release Candidate Number
+        run: |
+          version_with_rc=${RELEASE_CANDIDATE_TAG_NAME#apache-arrow-}
+          version=${version_with_rc%-rc*}
+          rc_num=${version_with_rc#${version}-rc}
+          echo "VERSION_WITH_RC=${version_with_rc}" >> ${GITHUB_ENV}
+          echo "VERSION=${version}" >> ${GITHUB_ENV}
+          echo "RC_NUM=${rc_num}" >> ${GITHUB_ENV}
+      - name: Download Release Candidate Artifacts 
+        run: |
+          mkdir release_candidate_artifacts
+          gh release download ${RELEASE_CANDIDATE_TAG_NAME} --repo apache/arrow --dir release_candidate_artifacts
+      - name: Create Release Title
+        run: |
+          title="Apache Arrow ${VERSION}"
+          echo "RELEASE_TITLE=${title}" >> ${GITHUB_ENV}
+      # Set the release notes to "TODO" temporarily. After the release notes page
+      # (https://arrow.apache.org/release/{VERSION}.html) is published, use 
+      # gh release edit to update the release notes to refer to the newly 
+      # pushed web page. See dev/post/post-05-update-gh-release-notes.sh
+      - name: Create GitHub Release
+        run: |
+          gh release create ${GITHUB_REF_NAME} \
+            --repo apache/arrow \
+            --verify-tag \
+            --title "${RELEASE_TITLE}" \
+            --notes "TODO" \
+            release_candidate_artifacts/*

.github/workflows/release_candidate.yml

@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Release
+
+on:
+  push:
+    tags:
+      # Trigger workflow when a tag whose name matches the pattern
+      # "apache-arrow-{MAJOR}.{MINOR}.{PATCH}-rc{RC_NUM}" is pushed.
+      - "apache-arrow-[0-9]+.[0-9]+.[0-9]+-rc[0-9]+"
+
+permissions:
+  contents: write
+
+env:
+  GH_TOKEN: ${{ github.token }}
+
+jobs:
+  publish:
+    name: Publish 
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps: 
+      - name: Checkout Arrow
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Store Version and Release Candidate Number
+        run: |
+          version_with_rc=${GITHUB_REF_NAME#apache-arrow-}
+          version=${version_with_rc%-rc*}
+          rc_num=${version_with_rc#${version}-rc}
+          echo "VERSION_WITH_RC=${version_with_rc}" >> ${GITHUB_ENV}
+          echo "VERSION=${version}" >> ${GITHUB_ENV}
+          echo "RC_NUM=${rc_num}" >> ${GITHUB_ENV}
+      - name: Create Release Candidate Title
+        run: |
+          title="Apache Arrow ${VERSION} RC${RC_NUM}"
+          echo "RELEASE_CANDIDATE_TITLE=${title}" >> ${GITHUB_ENV}
+      - name: Create Release Candidate Notes
+        run: |
+          release_notes="Release Candidate: ${VERSION} RC${RC_NUM}"
+          echo "RELEASE_CANDIDATE_NOTES=${release_notes}" >> ${GITHUB_ENV}
+      - name: Create Release tarball
+        run: |
+          cd dev/release/ && ./utils-create-release-tarball.sh ${VERSION} ${RC_NUM}
+          echo "RELEASE_TARBALL=apache-arrow-${VERSION}.tar.gz" >> ${GITHUB_ENV}
+      - name: Create GitHub Release
+        run: |
+          gh release create ${GITHUB_REF_NAME} \
+            --verify-tag \
+            --prerelease \
+            --title "${RELEASE_CANDIDATE_TITLE}" \
+            --notes "Release Notes: ${RELEASE_CANDIDATE_NOTES}" \
+            dev/release/${RELEASE_TARBALL}

.github/workflows/ruby.yml

@@ -323,6 +323,7 @@ jobs:
       ARROW_FLIGHT: ON
       ARROW_FLIGHT_SQL: ON
       ARROW_GANDIVA: OFF
+      ARROW_GLIB_VAPI: "false"
       ARROW_HDFS: OFF
       ARROW_HOME: "${{ github.workspace }}/dist"
       ARROW_JEMALLOC: OFF
@@ -345,6 +346,7 @@ jobs:
       CMAKE_UNITY_BUILD: ON
       VCPKG_BINARY_SOURCES: 'clear;nuget,GitHub,readwrite'
       VCPKG_ROOT: "${{ github.workspace }}/vcpkg"
+      VCPKG_TRIPLET: x64-windows
     permissions:
       packages: write
     steps:
@@ -405,7 +407,7 @@ jobs:
             -source "https://nuget.pkg.github.com/$GITHUB_REPOSITORY_OWNER/index.json"
       - name: Build C++ vcpkg dependencies
         run: |
-          vcpkg\vcpkg.exe install --triplet x64-windows --x-manifest-root cpp --x-install-root build\cpp\vcpkg_installed
+          vcpkg\vcpkg.exe install --triplet $env:VCPKG_TRIPLET --x-manifest-root cpp --x-install-root build\cpp\vcpkg_installed
       - name: Build C++
         shell: cmd
         run: |

.pre-commit-config.yaml

@@ -34,7 +34,13 @@ repos:
     hooks:
       - id: hadolint-docker
         name: Docker Format
-        exclude: ^dev/.*$
+        # We can enable this after we fix all existing lint failures.
+        # files: (/Dockerfile|\.dockerfile)$
+        files: >-
+          (
+          ?^ci/docker/python-wheel-windows-test-vs2019\.dockerfile$|
+          )
+        types: []
   - repo: https://github.com/pycqa/flake8
     rev: 6.1.0
     hooks:

c_glib/arrow-dataset-glib/enums.h.template

@@ -22,6 +22,8 @@
 
 #include <glib-object.h>
 
+#include <arrow-dataset-glib/version.h>
+
 G_BEGIN_DECLS
 /*** END file-header ***/
 
@@ -31,6 +33,7 @@ G_BEGIN_DECLS
 /*** END file-production ***/
 
 /*** BEGIN value-header ***/
+GADATASET_AVAILABLE_IN_ALL
 GType @enum_name@_get_type(void) G_GNUC_CONST;
 #define @ENUMPREFIX@_TYPE_@ENUMSHORT@ (@enum_name@_get_type())
 /*** END value-header ***/

c_glib/arrow-dataset-glib/meson.build

@@ -82,6 +82,7 @@ libarrow_dataset_glib = library('arrow-dataset-glib',
                                 implicit_include_directories: false,
                                 include_directories: base_include_directories,
                                 cpp_args: ['-DGADATASET_COMPILATION'],
+                                c_args: ['-DGADATASET_COMPILATION'],
                                 soversion: so_version,
                                 version: library_version)
 arrow_dataset_glib = declare_dependency(link_with: libarrow_dataset_glib,

c_glib/arrow-glib/enums.h.template

@@ -22,6 +22,8 @@
 
 #include <glib-object.h>
 
+#include <arrow-glib/version.h>
+
 G_BEGIN_DECLS
 /*** END file-header ***/
 
@@ -31,6 +33,7 @@ G_BEGIN_DECLS
 /*** END file-production ***/
 
 /*** BEGIN value-header ***/
+GARROW_AVAILABLE_IN_ALL
 GType @enum_name@_get_type(void) G_GNUC_CONST;
 #define @ENUMPREFIX@_TYPE_@ENUMSHORT@ (@enum_name@_get_type())
 /*** END value-header ***/

c_glib/arrow-glib/meson.build

@@ -250,6 +250,7 @@ libarrow_glib = library('arrow-glib',
                         implicit_include_directories: false,
                         include_directories: base_include_directories,
                         cpp_args: ['-DGARROW_COMPILATION'],
+                        c_args: ['-DGARROW_COMPILATION'],
                         soversion: so_version,
                         version: library_version)
 arrow_glib = declare_dependency(link_with: libarrow_glib,

c_glib/gandiva-glib/enums.h.template

@@ -22,6 +22,8 @@
 
 #include <glib-object.h>
 
+#include <gandiva-glib/version.h>
+
 G_BEGIN_DECLS
 /*** END file-header ***/
 
@@ -31,6 +33,7 @@ G_BEGIN_DECLS
 /*** END file-production ***/
 
 /*** BEGIN value-header ***/
+GGANDIVA_AVAILABLE_IN_ALL
 GType @enum_name@_get_type(void) G_GNUC_CONST;
 #define @ENUMPREFIX@_TYPE_@ENUMSHORT@ (@enum_name@_get_type())
 /*** END value-header ***/

c_glib/gandiva-glib/meson.build

@@ -86,6 +86,7 @@ libgandiva_glib = library('gandiva-glib',
                           implicit_include_directories: false,
                           include_directories: base_include_directories,
                           cpp_args: ['-DGGANDIVA_COMPILATION'],
+                          c_args: ['-DGGANDIVA_COMPILATION'],
                           soversion: so_version,
                           version: library_version)
 gandiva_glib = declare_dependency(link_with: libgandiva_glib,

c_glib/tool/generate-version-header.py

@@ -84,7 +84,7 @@ def write_header(
 
 
 def generate_visibility_macros(library: str) -> str:
-    return f"""#if (defined(_WIN32) || defined(__CYGWIN__)) && defined(_MSVC_LANG) && \
+    return f"""#if (defined(_WIN32) || defined(__CYGWIN__)) && defined(_MSC_VER) && \
   !defined({library}_STATIC_COMPILATION)
 #  define {library}_EXPORT __declspec(dllexport)
 #  define {library}_IMPORT __declspec(dllimport)

c_glib/vcpkg.json

@@ -3,6 +3,7 @@
   "version-string": "17.0.0-SNAPSHOT",
   "dependencies": [
     "glib",
+    "gobject-introspection",
     "pkgconf"
   ]
 }