Merge branch 'main' of https://github.com/CurtHagenlocher/arrow into …

…Union

.env

@@ -58,8 +58,8 @@ CUDA=11.0.3
 DASK=latest
 DOTNET=7.0
 GCC_VERSION=""
-GO=1.17
-STATICCHECK=v0.2.2
+GO=1.19.13
+STATICCHECK=v0.4.5
 HDFS=3.2.1
 JDK=8
 KARTOTHEK=latest

.github/workflows/go.yml

@@ -54,28 +54,23 @@ jobs:
         include:
           - arch-label: AMD64
             arch: amd64
-            go: 1.17
+            go: 1.19
             runs-on: ubuntu-latest
-            staticcheck: v0.2.2
           - arch-label: AMD64
             arch: amd64
-            go: 1.18
+            go: '1.20'
             runs-on: ubuntu-latest
-            staticcheck: v0.3.3
           - arch-label: ARM64
             arch: arm64v8
-            go: 1.17
-            staticcheck: v0.2.2
+            go: 1.19
             runs-on: ["self-hosted", "arm", "linux"]
           - arch-label: ARM64
             arch: arm64v8
-            go: 1.18
-            staticcheck: v0.3.3
+            go: '1.20'
             runs-on: ["self-hosted", "arm", "linux"]
     env:
       ARCH: ${{ matrix.arch }}
       GO: ${{ matrix.go }}
-      STATICCHECK: ${{ matrix.staticcheck }}
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v4
@@ -145,7 +140,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.18
+          go-version: 1.19
           cache: true
           cache-dependency-path: go/go.sum
       - name: Run build
@@ -161,15 +156,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: [1.17, 1.18]
-        include:
-          - go: 1.17
-            staticcheck: v0.2.2
-          - go: 1.18
-            staticcheck: v0.3.3
+        go: [1.19, '1.20']
     env:
       GO: ${{ matrix.go }}
-      STATICCHECK: ${{ matrix.staticcheck }}
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v4
@@ -208,15 +197,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: [1.17, 1.18]
-        include:
-          - go: 1.17
-            staticcheck: v0.2.2
-          - go: 1.18
-            staticcheck: v0.3.3
+        go: [1.19, '1.20']
     env:
       GO: ${{ matrix.go }}
-      STATICCHECK: ${{ matrix.staticcheck }}
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v4
@@ -253,12 +236,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: [1.17, 1.18]
-        include:
-          - go: 1.17
-            staticcheck: v0.2.2
-          - go: 1.18
-            staticcheck: v0.3.3
+        go: [1.19, '1.20']
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v4
@@ -272,7 +250,10 @@ jobs:
           cache: true
           cache-dependency-path: go/go.sum
       - name: Install staticcheck
-        run: go install honnef.co/go/tools/cmd/staticcheck@${{ matrix.staticcheck }}
+        shell: bash
+        run: |
+          . .env
+          go install honnef.co/go/tools/cmd/staticcheck@${STATICCHECK}
       - name: Build
         shell: bash
         run: ci/scripts/go_build.sh $(pwd)
@@ -288,12 +269,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: [1.17, 1.18]
-        include:
-          - go: 1.17
-            staticcheck: v0.2.2
-          - go: 1.18
-            staticcheck: v0.3.3
+        go: [1.19, '1.20']
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v4
@@ -306,8 +282,10 @@ jobs:
           go-version: ${{ matrix.go }}
           cache: true
           cache-dependency-path: go/go.sum
-      - name: Install staticcheck
-        run: go install honnef.co/go/tools/cmd/staticcheck@${{ matrix.staticcheck }}
+      - name: Install staticcheck      
+        run: |
+          . .env
+          go install honnef.co/go/tools/cmd/staticcheck@${STATICCHECK}
       - name: Build
         shell: bash
         run: ci/scripts/go_build.sh $(pwd)
@@ -349,12 +327,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: [1.17, 1.18]
-        include:
-          - go: 1.17
-            staticcheck: v0.2.2
-          - go: 1.18
-            staticcheck: v0.3.3
+        go: [1.19, '1.20']
     env:
       ARROW_GO_TESTCGO: "1"
     steps:
@@ -373,7 +346,9 @@ jobs:
         shell: bash
         run: brew install apache-arrow pkg-config
       - name: Install staticcheck
-        run: go install honnef.co/go/tools/cmd/staticcheck@${{ matrix.staticcheck }}
+        run: |
+          . .env
+          go install honnef.co/go/tools/cmd/staticcheck@${STATICCHECK}
       - name: Add To pkg config path
         shell: bash
         run: |
@@ -430,11 +405,14 @@ jobs:
       - name: Install go
         uses: actions/setup-go@v4
         with:
-          go-version: '1.18'
+          go-version: '1.19'
           cache: true
           cache-dependency-path: go/go.sum
       - name: Install staticcheck
-        run: go install honnef.co/go/tools/cmd/[email protected]
+        shell: bash
+        run: |
+          . .env
+          go install honnef.co/go/tools/cmd/staticcheck@${STATICCHECK}
       - name: Build
         shell: bash
         run: ci/scripts/go_build.sh $(pwd)

c_glib/test/test-array-datum.rb

@@ -61,7 +61,7 @@ def test_false
   end
 
   def test_to_string
-    assert_equal("Array", @datum.to_s)
+    assert_equal("Array([\n" + "  true,\n" + "  false\n" + "])", @datum.to_s)
   end
 
   def test_value

c_glib/test/test-chunked-array-datum.rb

@@ -49,7 +49,7 @@ def test_false
   end
 
   def test_to_string
-    assert_equal("ChunkedArray", @datum.to_s)
+    assert_equal("ChunkedArray([\n" + "  [\n" + "    true,\n" + "    false\n" + "  ]\n" + "])", @datum.to_s)
   end
 
   def test_value

c_glib/test/test-record-batch-datum.rb

@@ -49,7 +49,7 @@ def test_false
   end
 
   def test_to_string
-    assert_equal("RecordBatch", @datum.to_s)
+    assert_equal("RecordBatch(visible:   [\n" + "    true,\n" + "    false\n" + "  ]\n" + ")", @datum.to_s)
   end
 
   def test_value

c_glib/test/test-scalar-datum.rb

@@ -60,7 +60,7 @@ def test_false
   end
 
   def test_to_string
-    assert_equal("Scalar", @datum.to_s)
+    assert_equal("Scalar(true)", @datum.to_s)
   end
 
   def test_value

c_glib/test/test-table-datum.rb

@@ -49,7 +49,16 @@ def test_false
   end
 
   def test_to_string
-    assert_equal("Table", @datum.to_s)
+    assert_equal("Table(visible: bool\n" +
+    "----\n" +
+    "visible:\n" +
+    "  [\n" +
+    "    [\n" +
+    "      true,\n" +
+    "      false\n" +
+    "    ]\n" +
+    "  ]\n" +
+    ")", @datum.to_s)
   end
 
   def test_value

ci/docker/conda-integration.dockerfile

@@ -24,7 +24,7 @@ ARG maven=3.5
 ARG node=16
 ARG yarn=1.22
 ARG jdk=8
-ARG go=1.15
+ARG go=1.19.13
 
 # Install Archery and integration dependencies
 COPY ci/conda_env_archery.txt /arrow/ci/

ci/docker/debian-11-go.dockerfile

@@ -16,8 +16,8 @@
 # under the License.
 
 ARG arch=amd64
-ARG go=1.17
-ARG staticcheck=v0.2.2
+ARG go=1.19
+ARG staticcheck=v0.4.5
 FROM ${arch}/golang:${go}-bullseye
 
 # FROM collects all the args, get back the staticcheck version arg

ci/scripts/go_bench_adapt.py

@@ -20,7 +20,7 @@
 import uuid
 import logging
 from pathlib import Path
-from typing import List, Optional, Dict
+from typing import List
 
 from benchadapt import BenchmarkResult
 from benchadapt.adapters import BenchmarkAdapter
@@ -33,9 +33,9 @@
 
 # `github_commit_info` is meant to communicate GitHub-flavored commit
 # information to Conbench. See
-# https://github.com/conbench/conbench/blob/7c4968e631ecdc064559c86a1174a1353713b700/benchadapt/python/benchadapt/result.py#L66
+# https://github.com/conbench/conbench/blob/cf7931f/benchadapt/python/benchadapt/result.py#L66
 # for a specification.
-github_commit_info: Optional[Dict] = None
+github_commit_info = {"repository": "https://github.com/apache/arrow"}
 
 if os.environ.get("CONBENCH_REF") == "main":
     # Assume GitHub Actions CI. The environment variable lookups below are
@@ -53,7 +53,7 @@
 
     # This is probably a local dev environment, for testing. In this case, it
     # does usually not make sense to provide commit information (not a
-    # controlled CI environment). Explicitly keep `github_commit_info=None` to
+    # controlled CI environment). Explicitly leave out "commit" and "pr_number" to
     # reflect that (to not send commit information).
 
     # Reflect 'local dev' scenario in run_reason. Allow user to (optionally)
@@ -114,10 +114,9 @@ def _transform_results(self) -> List[BenchmarkResult]:
                     run_reason=run_reason,
                     github=github_commit_info,
                 )
-                if github_commit_info is not None:
-                    parsed.run_name = (
-                        f"{parsed.run_reason}: {github_commit_info['commit']}"
-                    )
+                parsed.run_name = (
+                    f"{parsed.run_reason}: {github_commit_info.get('commit')}"
+                )
                 parsed_results.append(parsed)
 
         return parsed_results

cpp/CMakeLists.txt

@@ -533,6 +533,9 @@ set(ARROW_TESTING_PC_LIBS "")
 set(ARROW_TESTING_PC_REQUIRES "")
 
 # For parquet.pc.
+set(PARQUET_PC_CFLAGS "")
+set(PARQUET_PC_CFLAGS_PRIVATE " -DPARQUET_STATIC")
+set(PARQUET_PC_REQUIRES "")
 set(PARQUET_PC_REQUIRES_PRIVATE "")
 
 include(ThirdpartyToolchain)

cpp/src/arrow/compute/api_aggregate.h

@@ -138,7 +138,7 @@ class ARROW_EXPORT QuantileOptions : public FunctionOptions {
   static constexpr char const kTypeName[] = "QuantileOptions";
   static QuantileOptions Defaults() { return QuantileOptions{}; }
 
-  /// quantile must be between 0 and 1 inclusive
+  /// probability level of quantile must be between 0 and 1 inclusive
   std::vector<double> q;
   enum Interpolation interpolation;
   /// If true (the default), null values are ignored. Otherwise, if any value is null,
@@ -162,7 +162,7 @@ class ARROW_EXPORT TDigestOptions : public FunctionOptions {
   static constexpr char const kTypeName[] = "TDigestOptions";
   static TDigestOptions Defaults() { return TDigestOptions{}; }
 
-  /// quantile must be between 0 and 1 inclusive
+  /// probability level of quantile must be between 0 and 1 inclusive
   std::vector<double> q;
   /// compression parameter, default 100
   uint32_t delta;

cpp/src/arrow/datum.cc

@@ -182,15 +182,15 @@ std::string Datum::ToString() const {
     case Datum::NONE:
       return "nullptr";
     case Datum::SCALAR:
-      return "Scalar";
+      return "Scalar(" + scalar()->ToString() + ")";
     case Datum::ARRAY:
-      return "Array";
+      return "Array(" + make_array()->ToString() + ")";
     case Datum::CHUNKED_ARRAY:
-      return "ChunkedArray";
+      return "ChunkedArray(" + chunked_array()->ToString() + ")";
     case Datum::RECORD_BATCH:
-      return "RecordBatch";
+      return "RecordBatch(" + record_batch()->ToString() + ")";
     case Datum::TABLE:
-      return "Table";
+      return "Table(" + table()->ToString() + ")";
     default:
       DCHECK(false);
       return "";

cpp/src/arrow/datum.h

@@ -301,7 +301,6 @@ struct ARROW_EXPORT Datum {
   bool operator==(const Datum& other) const { return Equals(other); }
   bool operator!=(const Datum& other) const { return !Equals(other); }
 
-  /// \brief Return a string representation of the kind of datum stored.
   std::string ToString() const;
 };
 

cpp/src/arrow/datum_test.cc

@@ -154,8 +154,8 @@ TEST(Datum, ToString) {
   Datum v1(arr);
   Datum v2(std::make_shared<Int8Scalar>(1));
 
-  ASSERT_EQ("Array", v1.ToString());
-  ASSERT_EQ("Scalar", v2.ToString());
+  ASSERT_EQ("Array([\n  1,\n  2,\n  3,\n  4\n])", v1.ToString());
+  ASSERT_EQ("Scalar(1)", v2.ToString());
 }
 
 TEST(Datum, TotalBufferSize) {

cpp/src/parquet/CMakeLists.txt

@@ -21,6 +21,17 @@ add_custom_target(parquet-benchmarks)
 add_custom_target(parquet-tests)
 add_dependencies(parquet-all parquet parquet-tests parquet-benchmarks)
 
+# If libparquet.a is only built, "pkg-config --cflags --libs parquet"
+# outputs build flags for static linking not shared
+# linking. PARQUET_PC_* except PARQUET_PC_*_PRIVATE are for the static
+# linking case.
+if(NOT ARROW_BUILD_SHARED AND ARROW_BUILD_STATIC)
+  string(APPEND PARQUET_PC_CFLAGS "${PARQUET_PC_CFLAGS_PRIVATE}")
+  set(PARQUET_PC_CFLAGS_PRIVATE "")
+  string(APPEND PARQUET_PC_REQUIRES "${PARQUET_PC_REQUIRES_PRIVATE}")
+  set(PARQUET_PC_REQUIRES_PRIVATE "")
+endif()
+
 function(ADD_PARQUET_TEST REL_TEST_NAME)
   set(one_value_args)
   set(multi_value_args EXTRA_DEPENDENCIES LABELS)

cpp/src/parquet/parquet.pc.in

@@ -26,8 +26,8 @@ full_so_version=@ARROW_FULL_SO_VERSION@
 Name: Apache Parquet
 Description: Apache Parquet is a columnar storage format.
 Version: @ARROW_VERSION@
-Requires: arrow
+Requires: arrow@PARQUET_PC_REQUIRES@
 Requires.private:@PARQUET_PC_REQUIRES_PRIVATE@
 Libs: -L${libdir} -lparquet
-Cflags: -I${includedir}
-Cflags.private: -DPARQUET_STATIC
+Cflags: -I${includedir}@PARQUET_PC_CFLAGS@
+Cflags.private:@PARQUET_PC_CFLAGS_PRIVATE@