From 54dfb82a401fa4c1e53fccb2f152b04f7bb85f4d Mon Sep 17 00:00:00 2001 From: Vibhatha Lakmal Abeykoon Date: Sat, 25 May 2024 07:25:11 +0530 Subject: [PATCH 01/93] GH-40933: [Java] Enhance the copyFrom* functionality in StringView (#41752) ### Rationale for this change Initial implementation of StringView doesn't contain `copy` functionality. This PR adds that feature. ### What changes are included in this PR? This PR adds `copyFrom` and `copyFromSafe` functions to `BaseVariableWidthViewVector`. ### Are these changes tested? Yes ### Are there any user-facing changes? No * GitHub Issue: #40933 Lead-authored-by: Vibhatha Abeykoon Co-authored-by: Vibhatha Lakmal Abeykoon Signed-off-by: David Li --- .../vector/BaseVariableWidthViewVector.java | 52 ++++- .../org/apache/arrow/vector/types/Types.java | 2 +- .../arrow/vector/TestVarCharViewVector.java | 197 ++++++++++++++++++ 3 files changed, 243 insertions(+), 8 deletions(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java index b3e86fab05462..aaa8098b690fd 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java @@ -33,6 +33,7 @@ import org.apache.arrow.memory.util.ByteFunctionHelpers; import org.apache.arrow.memory.util.CommonUtil; import org.apache.arrow.memory.util.hash.ArrowBufHasher; +import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.compare.VectorVisitor; import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.vector.types.pojo.Field; @@ -1334,30 +1335,67 @@ protected final void handleSafe(int index, int dataLength) { /** * Copy a cell value from a particular index in source vector to a particular position in this * vector. - * TODO: Improve functionality to support copying views. - * Enhance CopyFrom - * * @param fromIndex position to copy from in source vector * @param thisIndex position to copy to in this vector * @param from source vector */ @Override public void copyFrom(int fromIndex, int thisIndex, ValueVector from) { - throw new UnsupportedOperationException("copyFrom is not supported for VariableWidthVector"); + Preconditions.checkArgument(getMinorType() == from.getMinorType()); + if (from.isNull(fromIndex)) { + BitVectorHelper.unsetBit(validityBuffer, thisIndex); + } else { + final int viewLength = from.getDataBuffer().getInt((long) fromIndex * ELEMENT_SIZE); + BitVectorHelper.setBit(validityBuffer, thisIndex); + final int start = thisIndex * ELEMENT_SIZE; + final int copyStart = fromIndex * ELEMENT_SIZE; + from.getDataBuffer().getBytes(start, viewBuffer, copyStart, ELEMENT_SIZE); + if (viewLength > INLINE_SIZE) { + final int bufIndex = from.getDataBuffer().getInt(((long) fromIndex * ELEMENT_SIZE) + + LENGTH_WIDTH + PREFIX_WIDTH); + final int dataOffset = from.getDataBuffer().getInt(((long) fromIndex * ELEMENT_SIZE) + + LENGTH_WIDTH + PREFIX_WIDTH + BUF_INDEX_WIDTH); + final ArrowBuf dataBuf = ((BaseVariableWidthViewVector) from).dataBuffers.get(bufIndex); + final ArrowBuf thisDataBuf = allocateOrGetLastDataBuffer(viewLength); + thisDataBuf.setBytes(thisDataBuf.writerIndex(), dataBuf, dataOffset, viewLength); + thisDataBuf.writerIndex(thisDataBuf.writerIndex() + viewLength); + } + } + lastSet = thisIndex; } /** * Same as {@link #copyFrom(int, int, ValueVector)} except that it handles the case when the * capacity of the vector needs to be expanded before copy. - * TODO: Improve functionality to support copying views. - * Enhance CopyFrom * @param fromIndex position to copy from in source vector * @param thisIndex position to copy to in this vector * @param from source vector */ @Override public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { - throw new UnsupportedOperationException("copyFromSafe is not supported for VariableWidthVector"); + Preconditions.checkArgument(getMinorType() == from.getMinorType()); + if (from.isNull(fromIndex)) { + handleSafe(thisIndex, 0); + BitVectorHelper.unsetBit(validityBuffer, thisIndex); + } else { + final int viewLength = from.getDataBuffer().getInt((long) fromIndex * ELEMENT_SIZE); + handleSafe(thisIndex, viewLength); + BitVectorHelper.setBit(validityBuffer, thisIndex); + final int start = thisIndex * ELEMENT_SIZE; + final int copyStart = fromIndex * ELEMENT_SIZE; + from.getDataBuffer().getBytes(start, viewBuffer, copyStart, ELEMENT_SIZE); + if (viewLength > INLINE_SIZE) { + final int bufIndex = from.getDataBuffer().getInt(((long) fromIndex * ELEMENT_SIZE) + + LENGTH_WIDTH + PREFIX_WIDTH); + final int dataOffset = from.getDataBuffer().getInt(((long) fromIndex * ELEMENT_SIZE) + + LENGTH_WIDTH + PREFIX_WIDTH + BUF_INDEX_WIDTH); + final ArrowBuf dataBuf = ((BaseVariableWidthViewVector) from).dataBuffers.get(bufIndex); + final ArrowBuf thisDataBuf = allocateOrGetLastDataBuffer(viewLength); + thisDataBuf.setBytes(thisDataBuf.writerIndex(), dataBuf, dataOffset, viewLength); + thisDataBuf.writerIndex(thisDataBuf.writerIndex() + viewLength); + } + } + lastSet = thisIndex; } @Override diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java index e10a65e3b2c53..abed4d1ff0143 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java @@ -568,7 +568,7 @@ public FieldWriter getNewFieldWriter(ValueVector vector) { return new VarBinaryWriterImpl((VarBinaryVector) vector); } }, - VIEWVARBINARY(Binary.INSTANCE) { + VIEWVARBINARY(BinaryView.INSTANCE) { @Override public FieldVector getNewVector( Field field, diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharViewVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharViewVector.java index 2d37b0b4eb9ad..17bc08c7d398c 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharViewVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharViewVector.java @@ -36,6 +36,8 @@ import java.util.List; import java.util.Objects; import java.util.Random; +import java.util.function.Function; +import java.util.stream.Stream; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; @@ -52,6 +54,9 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; public class TestVarCharViewVector { @@ -1517,6 +1522,198 @@ public void testVectorLoadUnload() { } } + static Stream vectorCreatorProvider() { + return Stream.of( + Arguments.of((Function) + (allocator -> newVector(ViewVarBinaryVector.class, EMPTY_SCHEMA_PATH, + Types.MinorType.VIEWVARBINARY, allocator))), + Arguments.of((Function) + (allocator -> newVector(ViewVarCharVector.class, EMPTY_SCHEMA_PATH, + Types.MinorType.VIEWVARCHAR, allocator))) + ); + } + + @ParameterizedTest + @MethodSource({"vectorCreatorProvider"}) + public void testCopyFromWithNulls(Function vectorCreator) { + try (final BaseVariableWidthViewVector vector = vectorCreator.apply(allocator); + final BaseVariableWidthViewVector vector2 = vectorCreator.apply(allocator)) { + final int initialCapacity = 1024; + vector.setInitialCapacity(initialCapacity); + vector.allocateNew(); + int capacity = vector.getValueCapacity(); + assertTrue(capacity >= initialCapacity); + + // setting number of values such that we have enough space in the initial allocation + // to avoid re-allocation. This is to test copyFrom() without re-allocation. + final int numberOfValues = initialCapacity / 2 / ViewVarCharVector.ELEMENT_SIZE; + + final String prefixString = generateRandomString(12); + + for (int i = 0; i < numberOfValues; i++) { + if (i % 3 == 0) { + // null values + vector.setNull(i); + } else if (i % 3 == 1) { + // short strings + byte[] b = Integer.toString(i).getBytes(StandardCharsets.UTF_8); + vector.set(i, b, 0, b.length); + } else { + // long strings + byte[] b = (i + prefixString).getBytes(StandardCharsets.UTF_8); + vector.set(i, b, 0, b.length); + } + } + + assertEquals(capacity, vector.getValueCapacity()); + + vector.setValueCount(numberOfValues); + + for (int i = 0; i < numberOfValues; i++) { + if (i % 3 == 0) { + assertNull(vector.getObject(i)); + } else if (i % 3 == 1) { + assertArrayEquals(Integer.toString(i).getBytes(StandardCharsets.UTF_8), + vector.get(i), + "unexpected value at index: " + i); + } else { + assertArrayEquals((i + prefixString).getBytes(StandardCharsets.UTF_8), + vector.get(i), + "unexpected value at index: " + i); + } + } + + vector2.setInitialCapacity(initialCapacity); + vector2.allocateNew(); + int capacity2 = vector2.getValueCapacity(); + assertEquals(capacity2, capacity); + + for (int i = 0; i < numberOfValues; i++) { + vector2.copyFrom(i, i, vector); + if (i % 3 == 0) { + assertNull(vector2.getObject(i)); + } else if (i % 3 == 1) { + assertArrayEquals(Integer.toString(i).getBytes(StandardCharsets.UTF_8), + vector.get(i), + "unexpected value at index: " + i); + } else { + assertArrayEquals((i + prefixString).getBytes(StandardCharsets.UTF_8), + vector.get(i), + "unexpected value at index: " + i); + } + } + + assertEquals(capacity, vector2.getValueCapacity()); + + vector2.setValueCount(numberOfValues); + + for (int i = 0; i < numberOfValues; i++) { + if (i % 3 == 0) { + assertNull(vector2.getObject(i)); + } else if (i % 3 == 1) { + assertArrayEquals(Integer.toString(i).getBytes(StandardCharsets.UTF_8), + vector.get(i), + "unexpected value at index: " + i); + } else { + assertArrayEquals((i + prefixString).getBytes(StandardCharsets.UTF_8), + vector.get(i), + "unexpected value at index: " + i); + } + } + } + } + + @ParameterizedTest + @MethodSource("vectorCreatorProvider") + public void testCopyFromSafeWithNulls(Function vectorCreator) { + try (final BaseVariableWidthViewVector vector = vectorCreator.apply(allocator); + final BaseVariableWidthViewVector vector2 = vectorCreator.apply(allocator)) { + + final int initialCapacity = 4096; + vector.setInitialCapacity(initialCapacity); + vector.allocateNew(); + int capacity = vector.getValueCapacity(); + assertTrue(capacity >= initialCapacity); + + final int numberOfValues = initialCapacity / ViewVarCharVector.ELEMENT_SIZE; + + final String prefixString = generateRandomString(12); + + for (int i = 0; i < numberOfValues; i++) { + if (i % 3 == 0) { + // null values + vector.setNull(i); + } else if (i % 3 == 1) { + // short strings + byte[] b = Integer.toString(i).getBytes(StandardCharsets.UTF_8); + vector.setSafe(i, b, 0, b.length); + } else { + // long strings + byte[] b = (i + prefixString).getBytes(StandardCharsets.UTF_8); + vector.setSafe(i, b, 0, b.length); + } + } + + /* NO reAlloc() should have happened in setSafe() */ + assertEquals(capacity, vector.getValueCapacity()); + + vector.setValueCount(numberOfValues); + + for (int i = 0; i < numberOfValues; i++) { + if (i % 3 == 0) { + assertNull(vector.getObject(i)); + } else if (i % 3 == 1) { + assertArrayEquals(Integer.toString(i).getBytes(StandardCharsets.UTF_8), + vector.get(i), + "unexpected value at index: " + i); + } else { + assertArrayEquals((i + prefixString).getBytes(StandardCharsets.UTF_8), + vector.get(i), + "unexpected value at index: " + i); + } + } + + vector2.setInitialCapacity(initialCapacity); + vector2.allocateNew(); + int capacity2 = vector2.getValueCapacity(); + assertEquals(capacity2, capacity); + + for (int i = 0; i < numberOfValues; i++) { + vector2.copyFromSafe(i, i, vector); + if (i % 3 == 0) { + assertNull(vector2.getObject(i)); + } else if (i % 3 == 1) { + assertArrayEquals(Integer.toString(i).getBytes(StandardCharsets.UTF_8), + vector.get(i), + "unexpected value at index: " + i); + } else { + assertArrayEquals((i + prefixString).getBytes(StandardCharsets.UTF_8), + vector.get(i), + "unexpected value at index: " + i); + } + } + + /* NO reAlloc() should have happened in setSafe() */ + assertEquals(capacity, vector2.getValueCapacity()); + + vector2.setValueCount(numberOfValues); + + for (int i = 0; i < numberOfValues; i++) { + if (i % 3 == 0) { + assertNull(vector2.getObject(i)); + } else if (i % 3 == 1) { + assertArrayEquals(Integer.toString(i).getBytes(StandardCharsets.UTF_8), + vector.get(i), + "unexpected value at index: " + i); + } else { + assertArrayEquals((i + prefixString).getBytes(StandardCharsets.UTF_8), + vector.get(i), + "unexpected value at index: " + i); + } + } + } + } + private String generateRandomString(int length) { Random random = new Random(); StringBuilder sb = new StringBuilder(length); From ad711ec4590170ad225f2cbb7b6a1113a87c6e67 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 25 May 2024 11:06:19 +0900 Subject: [PATCH 02/93] MINOR: [Java] Bump checker.framework.version from 3.42.0 to 3.43.0 in /java (#41520) Bumps `checker.framework.version` from 3.42.0 to 3.43.0. Updates `org.checkerframework:checker-qual` from 3.42.0 to 3.43.0
Release notes

Sourced from org.checkerframework:checker-qual's releases.

Checker Framework 3.43.0

Version 3.43.0 (May 1, 2024)

User-visible changes:

Method, constructor, lambda, and method reference type inference has been greatly improved. The -AconservativeUninferredTypeArguments option is no longer necessary and has been removed.

Renamed command-line arguments:

  • -AskipDirs has been renamed to -AskipFiles. -AskipDirs will continue to work for the time being.

New command-line arguments:

  • -AonlyFiles complements -AskipFiles

A specialized inference algorithm for the Resource Leak Checker runs automatically as part of whole-program inference.

Implementation details:

Deprecated ObjectCreationNode#getConstructor in favor of new ObjectCreationNode#getTypeToInstantiate().

Renamed AbstractCFGVisualizer.visualizeBlockHelper() to visualizeBlockWithSeparator().

Moved methods from TreeUtils to subclasses of TreeUtilsAfterJava11:

  • isConstantCaseLabelTree
  • isDefaultCaseLabelTree
  • isPatternCaseLabelTree

Renamed BaseTypeVisitor.checkForPolymorphicQualifiers() to warnInvalidPolymorphicQualifier().

Closed issues:

#979, #4559, #4593, #5058, #5734, #5781, #6071, #6093, #6239, #6297, #6317, #6322, #6346, #6373, #6376, #6378, #6379, #6380, #6389, #6393, #6396, #6402, #6406, #6407, #6417, #6421, #6430, #6433, #6438, #6442, #6473, #6480, #6507, #6531, #6535.

Changelog

Sourced from org.checkerframework:checker-qual's changelog.

Version 3.43.0 (May 1, 2024)

User-visible changes:

Method, constructor, lambda, and method reference type inference has been greatly improved. The -AconservativeUninferredTypeArguments option is no longer necessary and has been removed.

Renamed command-line arguments:

  • -AskipDirs has been renamed to -AskipFiles. -AskipDirs will continue to work for the time being.

New command-line arguments:

  • -AonlyFiles complements -AskipFiles

A specialized inference algorithm for the Resource Leak Checker runs automatically as part of whole-program inference.

Implementation details:

Deprecated ObjectCreationNode#getConstructor in favor of new ObjectCreationNode#getTypeToInstantiate().

Renamed AbstractCFGVisualizer.visualizeBlockHelper() to visualizeBlockWithSeparator().

Moved methods from TreeUtils to subclasses of TreeUtilsAfterJava11:

  • isConstantCaseLabelTree
  • isDefaultCaseLabelTree
  • isPatternCaseLabelTree

Renamed BaseTypeVisitor.checkForPolymorphicQualifiers() to warnInvalidPolymorphicQualifier().

Closed issues:

#979, #4559, #4593, #5058, #5734, #5781, #6071, #6093, #6239, #6297, #6317, #6322, #6346, #6373, #6376, #6378, #6379, #6380, #6389, #6393, #6396, #6402, #6406, #6407, #6417, #6421, #6430, #6433, #6438, #6442, #6473, #6480, #6507, #6531, #6535.

Commits

Updates `org.checkerframework:checker` from 3.42.0 to 3.43.0
Release notes

Sourced from org.checkerframework:checker's releases.

Checker Framework 3.43.0

Version 3.43.0 (May 1, 2024)

User-visible changes:

Method, constructor, lambda, and method reference type inference has been greatly improved. The -AconservativeUninferredTypeArguments option is no longer necessary and has been removed.

Renamed command-line arguments:

  • -AskipDirs has been renamed to -AskipFiles. -AskipDirs will continue to work for the time being.

New command-line arguments:

  • -AonlyFiles complements -AskipFiles

A specialized inference algorithm for the Resource Leak Checker runs automatically as part of whole-program inference.

Implementation details:

Deprecated ObjectCreationNode#getConstructor in favor of new ObjectCreationNode#getTypeToInstantiate().

Renamed AbstractCFGVisualizer.visualizeBlockHelper() to visualizeBlockWithSeparator().

Moved methods from TreeUtils to subclasses of TreeUtilsAfterJava11:

  • isConstantCaseLabelTree
  • isDefaultCaseLabelTree
  • isPatternCaseLabelTree

Renamed BaseTypeVisitor.checkForPolymorphicQualifiers() to warnInvalidPolymorphicQualifier().

Closed issues:

#979, #4559, #4593, #5058, #5734, #5781, #6071, #6093, #6239, #6297, #6317, #6322, #6346, #6373, #6376, #6378, #6379, #6380, #6389, #6393, #6396, #6402, #6406, #6407, #6417, #6421, #6430, #6433, #6438, #6442, #6473, #6480, #6507, #6531, #6535.

Changelog

Sourced from org.checkerframework:checker's changelog.

Version 3.43.0 (May 1, 2024)

User-visible changes:

Method, constructor, lambda, and method reference type inference has been greatly improved. The -AconservativeUninferredTypeArguments option is no longer necessary and has been removed.

Renamed command-line arguments:

  • -AskipDirs has been renamed to -AskipFiles. -AskipDirs will continue to work for the time being.

New command-line arguments:

  • -AonlyFiles complements -AskipFiles

A specialized inference algorithm for the Resource Leak Checker runs automatically as part of whole-program inference.

Implementation details:

Deprecated ObjectCreationNode#getConstructor in favor of new ObjectCreationNode#getTypeToInstantiate().

Renamed AbstractCFGVisualizer.visualizeBlockHelper() to visualizeBlockWithSeparator().

Moved methods from TreeUtils to subclasses of TreeUtilsAfterJava11:

  • isConstantCaseLabelTree
  • isDefaultCaseLabelTree
  • isPatternCaseLabelTree

Renamed BaseTypeVisitor.checkForPolymorphicQualifiers() to warnInvalidPolymorphicQualifier().

Closed issues:

#979, #4559, #4593, #5058, #5734, #5781, #6071, #6093, #6239, #6297, #6317, #6322, #6346, #6373, #6376, #6378, #6379, #6380, #6389, #6393, #6396, #6402, #6406, #6407, #6417, #6421, #6430, #6433, #6438, #6442, #6473, #6480, #6507, #6531, #6535.

Commits

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: David Li --- java/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/pom.xml b/java/pom.xml index 925ec585152bc..289810daba3ac 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -98,7 +98,7 @@ 3.12.1 5.11.0 5.2.0 - 3.42.0 + 3.43.0 From 7c8ce4589ae9e3c4a9c0cd54cff81a54ac003079 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sat, 25 May 2024 12:34:04 +0900 Subject: [PATCH 03/93] GH-41770: [CI][GLib] Remove temporary files explicitly (#41807) ### Rationale for this change If we remove temporary files by GC, "`unlink': Permission denied" warnings are happen on Windows. ### What changes are included in this PR? Use `Tempfile.create {...}` to remove temporary files explicitly. ### Are these changes tested? Yes. ### Are there any user-facing changes? No. * GitHub Issue: #41770 Authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- c_glib/test/parquet/test-arrow-file-reader.rb | 27 +++++--- c_glib/test/parquet/test-arrow-file-writer.rb | 27 +++++--- .../test/parquet/test-boolean-statistics.rb | 24 +++++--- .../parquet/test-byte-array-statistics.rb | 24 +++++--- .../parquet/test-column-chunk-metadata.rb | 61 +++++++++++-------- c_glib/test/parquet/test-double-statistics.rb | 24 +++++--- c_glib/test/parquet/test-file-metadata.rb | 61 +++++++++++-------- ...test-fixed-length-byte-array-statistics.rb | 28 ++++++--- c_glib/test/parquet/test-float-statistics.rb | 24 +++++--- c_glib/test/parquet/test-int32-statistics.rb | 24 +++++--- c_glib/test/parquet/test-int64-statistics.rb | 26 +++++--- .../test/parquet/test-row-group-metadata.rb | 61 +++++++++++-------- c_glib/test/parquet/test-statistics.rb | 36 +++++++---- 13 files changed, 281 insertions(+), 166 deletions(-) diff --git a/c_glib/test/parquet/test-arrow-file-reader.rb b/c_glib/test/parquet/test-arrow-file-reader.rb index 45eb335965434..eff5ad966aea6 100644 --- a/c_glib/test/parquet/test-arrow-file-reader.rb +++ b/c_glib/test/parquet/test-arrow-file-reader.rb @@ -20,16 +20,23 @@ class TestParquetArrowFileReader < Test::Unit::TestCase def setup omit("Parquet is required") unless defined?(::Parquet) - @file = Tempfile.open(["data", ".parquet"]) - @a_array = build_string_array(["foo", "bar"]) - @b_array = build_int32_array([123, 456]) - @table = build_table("a" => @a_array, - "b" => @b_array) - writer = Parquet::ArrowFileWriter.new(@table.schema, @file.path) - chunk_size = 1 - writer.write_table(@table, chunk_size) - writer.close - @reader = Parquet::ArrowFileReader.new(@file.path) + Tempfile.create(["data", ".parquet"]) do |file| + @file = file + @a_array = build_string_array(["foo", "bar"]) + @b_array = build_int32_array([123, 456]) + @table = build_table("a" => @a_array, + "b" => @b_array) + writer = Parquet::ArrowFileWriter.new(@table.schema, @file.path) + chunk_size = 1 + writer.write_table(@table, chunk_size) + writer.close + @reader = Parquet::ArrowFileReader.new(@file.path) + begin + yield + ensure + @reader.unref + end + end end def test_schema diff --git a/c_glib/test/parquet/test-arrow-file-writer.rb b/c_glib/test/parquet/test-arrow-file-writer.rb index 855527444d063..f899e7273b2a2 100644 --- a/c_glib/test/parquet/test-arrow-file-writer.rb +++ b/c_glib/test/parquet/test-arrow-file-writer.rb @@ -20,7 +20,10 @@ class TestParquetArrowFileWriter < Test::Unit::TestCase def setup omit("Parquet is required") unless defined?(::Parquet) - @file = Tempfile.open(["data", ".parquet"]) + Tempfile.create(["data", ".parquet"]) do |file| + @file = file + yield + end end def test_write @@ -33,14 +36,18 @@ def test_write writer.close reader = Parquet::ArrowFileReader.new(@file.path) - reader.use_threads = true - assert_equal([ - enabled_values.length / chunk_size, - true, - ], - [ - reader.n_row_groups, - table.equal_metadata(reader.read_table, false), - ]) + begin + reader.use_threads = true + assert_equal([ + enabled_values.length / chunk_size, + true, + ], + [ + reader.n_row_groups, + table.equal_metadata(reader.read_table, false), + ]) + ensure + reader.unref + end end end diff --git a/c_glib/test/parquet/test-boolean-statistics.rb b/c_glib/test/parquet/test-boolean-statistics.rb index 6131a22195cb8..244348641320e 100644 --- a/c_glib/test/parquet/test-boolean-statistics.rb +++ b/c_glib/test/parquet/test-boolean-statistics.rb @@ -20,14 +20,22 @@ class TestParquetBooleanStatistics < Test::Unit::TestCase def setup omit("Parquet is required") unless defined?(::Parquet) - @file = Tempfile.open(["data", ".parquet"]) - @table = build_table("boolean" => build_boolean_array([nil, false, true])) - writer = Parquet::ArrowFileWriter.new(@table.schema, @file.path) - chunk_size = 1024 - writer.write_table(@table, chunk_size) - writer.close - reader = Parquet::ArrowFileReader.new(@file.path) - @statistics = reader.metadata.get_row_group(0).get_column_chunk(0).statistics + Tempfile.create(["data", ".parquet"]) do |file| + @file = file + @table = build_table("boolean" => build_boolean_array([nil, false, true])) + writer = Parquet::ArrowFileWriter.new(@table.schema, @file.path) + chunk_size = 1024 + writer.write_table(@table, chunk_size) + writer.close + reader = Parquet::ArrowFileReader.new(@file.path) + begin + @statistics = + reader.metadata.get_row_group(0).get_column_chunk(0).statistics + yield + ensure + reader.unref + end + end end test("#min") do diff --git a/c_glib/test/parquet/test-byte-array-statistics.rb b/c_glib/test/parquet/test-byte-array-statistics.rb index 50ec409dbce7c..b9693a77fff13 100644 --- a/c_glib/test/parquet/test-byte-array-statistics.rb +++ b/c_glib/test/parquet/test-byte-array-statistics.rb @@ -20,14 +20,22 @@ class TestParquetByteArrayStatistics < Test::Unit::TestCase def setup omit("Parquet is required") unless defined?(::Parquet) - @file = Tempfile.open(["data", ".parquet"]) - @table = build_table("string" => build_string_array([nil, "abc", "xyz"])) - writer = Parquet::ArrowFileWriter.new(@table.schema, @file.path) - chunk_size = 1024 - writer.write_table(@table, chunk_size) - writer.close - reader = Parquet::ArrowFileReader.new(@file.path) - @statistics = reader.metadata.get_row_group(0).get_column_chunk(0).statistics + Tempfile.create(["data", ".parquet"]) do |file| + @file = file + @table = build_table("string" => build_string_array([nil, "abc", "xyz"])) + writer = Parquet::ArrowFileWriter.new(@table.schema, @file.path) + chunk_size = 1024 + writer.write_table(@table, chunk_size) + writer.close + reader = Parquet::ArrowFileReader.new(@file.path) + begin + @statistics = + reader.metadata.get_row_group(0).get_column_chunk(0).statistics + yield + ensure + reader.unref + end + end end test("#min") do diff --git a/c_glib/test/parquet/test-column-chunk-metadata.rb b/c_glib/test/parquet/test-column-chunk-metadata.rb index a93fe85bbfbf1..f0012f0124577 100644 --- a/c_glib/test/parquet/test-column-chunk-metadata.rb +++ b/c_glib/test/parquet/test-column-chunk-metadata.rb @@ -20,35 +20,46 @@ class TestParquetColumnChunkMetadata < Test::Unit::TestCase def setup omit("Parquet is required") unless defined?(::Parquet) - @file = Tempfile.open(["data", ".parquet"]) - @string_array = build_string_array([nil, "hello"]) - fields = [ - Arrow::Field.new("int8", Arrow::Int8DataType.new), - Arrow::Field.new("boolean", Arrow::BooleanDataType.new), - ] - structs = [ - { - "int8" => -29, - "boolean" => true, - }, - nil, - ] - @struct_array = build_struct_array(fields, structs) - @table = build_table("string" => @string_array, - "struct" => @struct_array) - writer = Parquet::ArrowFileWriter.new(@table.schema, @file.path) - chunk_size = 1 - writer.write_table(@table, chunk_size) - writer.close - reader = Parquet::ArrowFileReader.new(@file.path) - @metadata = reader.metadata.get_row_group(0).get_column_chunk(0) + Tempfile.create(["data", ".parquet"]) do |file| + @file = file + @string_array = build_string_array([nil, "hello"]) + fields = [ + Arrow::Field.new("int8", Arrow::Int8DataType.new), + Arrow::Field.new("boolean", Arrow::BooleanDataType.new), + ] + structs = [ + { + "int8" => -29, + "boolean" => true, + }, + nil, + ] + @struct_array = build_struct_array(fields, structs) + @table = build_table("string" => @string_array, + "struct" => @struct_array) + writer = Parquet::ArrowFileWriter.new(@table.schema, @file.path) + chunk_size = 1 + writer.write_table(@table, chunk_size) + writer.close + reader = Parquet::ArrowFileReader.new(@file.path) + begin + @metadata = reader.metadata.get_row_group(0).get_column_chunk(0) + yield + ensure + reader.unref + end + end end test("#==") do reader = Parquet::ArrowFileReader.new(@file.path) - other_metadata = reader.metadata.get_row_group(0).get_column_chunk(0) - assert do - @metadata == other_metadata + begin + other_metadata = reader.metadata.get_row_group(0).get_column_chunk(0) + assert do + @metadata == other_metadata + end + ensure + reader.unref end end diff --git a/c_glib/test/parquet/test-double-statistics.rb b/c_glib/test/parquet/test-double-statistics.rb index a610fb24a9bdf..6c7a95824570d 100644 --- a/c_glib/test/parquet/test-double-statistics.rb +++ b/c_glib/test/parquet/test-double-statistics.rb @@ -20,14 +20,22 @@ class TestParquetDoubleStatistics < Test::Unit::TestCase def setup omit("Parquet is required") unless defined?(::Parquet) - @file = Tempfile.open(["data", ".parquet"]) - @table = build_table("double" => build_double_array([nil, -2.9, 2.9])) - writer = Parquet::ArrowFileWriter.new(@table.schema, @file.path) - chunk_size = 1024 - writer.write_table(@table, chunk_size) - writer.close - reader = Parquet::ArrowFileReader.new(@file.path) - @statistics = reader.metadata.get_row_group(0).get_column_chunk(0).statistics + Tempfile.create(["data", ".parquet"]) do |file| + @file = file + @table = build_table("double" => build_double_array([nil, -2.9, 2.9])) + writer = Parquet::ArrowFileWriter.new(@table.schema, @file.path) + chunk_size = 1024 + writer.write_table(@table, chunk_size) + writer.close + reader = Parquet::ArrowFileReader.new(@file.path) + begin + @statistics = + reader.metadata.get_row_group(0).get_column_chunk(0).statistics + yield + ensure + reader.unref + end + end end test("#min") do diff --git a/c_glib/test/parquet/test-file-metadata.rb b/c_glib/test/parquet/test-file-metadata.rb index 2bca7e66e0b07..aec3f4ab829b9 100644 --- a/c_glib/test/parquet/test-file-metadata.rb +++ b/c_glib/test/parquet/test-file-metadata.rb @@ -20,35 +20,46 @@ class TestParquetFileMetadata < Test::Unit::TestCase def setup omit("Parquet is required") unless defined?(::Parquet) - @file = Tempfile.open(["data", ".parquet"]) - @string_array = build_string_array([nil, "hello"]) - fields = [ - Arrow::Field.new("int8", Arrow::Int8DataType.new), - Arrow::Field.new("boolean", Arrow::BooleanDataType.new), - ] - structs = [ - { - "int8" => -29, - "boolean" => true, - }, - nil, - ] - @struct_array = build_struct_array(fields, structs) - @table = build_table("string" => @string_array, - "struct" => @struct_array) - writer = Parquet::ArrowFileWriter.new(@table.schema, @file.path) - chunk_size = 1 - writer.write_table(@table, chunk_size) - writer.close - reader = Parquet::ArrowFileReader.new(@file.path) - @metadata = reader.metadata + Tempfile.create(["data", ".parquet"]) do |file| + @file = file + @string_array = build_string_array([nil, "hello"]) + fields = [ + Arrow::Field.new("int8", Arrow::Int8DataType.new), + Arrow::Field.new("boolean", Arrow::BooleanDataType.new), + ] + structs = [ + { + "int8" => -29, + "boolean" => true, + }, + nil, + ] + @struct_array = build_struct_array(fields, structs) + @table = build_table("string" => @string_array, + "struct" => @struct_array) + writer = Parquet::ArrowFileWriter.new(@table.schema, @file.path) + chunk_size = 1 + writer.write_table(@table, chunk_size) + writer.close + reader = Parquet::ArrowFileReader.new(@file.path) + begin + @metadata = reader.metadata + yield + ensure + reader.unref + end + end end test("#==") do reader = Parquet::ArrowFileReader.new(@file.path) - other_metadata = reader.metadata - assert do - @metadata == other_metadata + begin + other_metadata = reader.metadata + assert do + @metadata == other_metadata + end + ensure + reader.unref end end diff --git a/c_glib/test/parquet/test-fixed-length-byte-array-statistics.rb b/c_glib/test/parquet/test-fixed-length-byte-array-statistics.rb index 87a96d009c509..c2f179627d06a 100644 --- a/c_glib/test/parquet/test-fixed-length-byte-array-statistics.rb +++ b/c_glib/test/parquet/test-fixed-length-byte-array-statistics.rb @@ -20,16 +20,24 @@ class TestParquetFixedLengthByteArrayStatistics < Test::Unit::TestCase def setup omit("Parquet is required") unless defined?(::Parquet) - @file = Tempfile.open(["data", ".parquet"]) - data_type = Arrow::FixedSizeBinaryDataType.new(3) - array = build_fixed_size_binary_array(data_type, [nil, "abc", "xyz"]) - @table = build_table("binary" => array) - writer = Parquet::ArrowFileWriter.new(@table.schema, @file.path) - chunk_size = 1024 - writer.write_table(@table, chunk_size) - writer.close - reader = Parquet::ArrowFileReader.new(@file.path) - @statistics = reader.metadata.get_row_group(0).get_column_chunk(0).statistics + Tempfile.create(["data", ".parquet"]) do |file| + @file = file + data_type = Arrow::FixedSizeBinaryDataType.new(3) + array = build_fixed_size_binary_array(data_type, [nil, "abc", "xyz"]) + @table = build_table("binary" => array) + writer = Parquet::ArrowFileWriter.new(@table.schema, @file.path) + chunk_size = 1024 + writer.write_table(@table, chunk_size) + writer.close + reader = Parquet::ArrowFileReader.new(@file.path) + begin + @statistics = + reader.metadata.get_row_group(0).get_column_chunk(0).statistics + yield + ensure + reader.unref + end + end end test("#min") do diff --git a/c_glib/test/parquet/test-float-statistics.rb b/c_glib/test/parquet/test-float-statistics.rb index 2622a2bb36fe6..7d1a233f53ca0 100644 --- a/c_glib/test/parquet/test-float-statistics.rb +++ b/c_glib/test/parquet/test-float-statistics.rb @@ -20,14 +20,22 @@ class TestParquetFloatStatistics < Test::Unit::TestCase def setup omit("Parquet is required") unless defined?(::Parquet) - @file = Tempfile.open(["data", ".parquet"]) - @table = build_table("float" => build_float_array([nil, -2.9, 2.9])) - writer = Parquet::ArrowFileWriter.new(@table.schema, @file.path) - chunk_size = 1024 - writer.write_table(@table, chunk_size) - writer.close - reader = Parquet::ArrowFileReader.new(@file.path) - @statistics = reader.metadata.get_row_group(0).get_column_chunk(0).statistics + Tempfile.create(["data", ".parquet"]) do |file| + @file = file + @table = build_table("float" => build_float_array([nil, -2.9, 2.9])) + writer = Parquet::ArrowFileWriter.new(@table.schema, @file.path) + chunk_size = 1024 + writer.write_table(@table, chunk_size) + writer.close + reader = Parquet::ArrowFileReader.new(@file.path) + begin + @statistics = + reader.metadata.get_row_group(0).get_column_chunk(0).statistics + yield + ensure + reader.unref + end + end end test("#min") do diff --git a/c_glib/test/parquet/test-int32-statistics.rb b/c_glib/test/parquet/test-int32-statistics.rb index 041f07c74292f..8d41327f88014 100644 --- a/c_glib/test/parquet/test-int32-statistics.rb +++ b/c_glib/test/parquet/test-int32-statistics.rb @@ -20,14 +20,22 @@ class TestParquetInt32Statistics < Test::Unit::TestCase def setup omit("Parquet is required") unless defined?(::Parquet) - @file = Tempfile.open(["data", ".parquet"]) - @table = build_table("int32" => build_int32_array([nil, -2, 9])) - writer = Parquet::ArrowFileWriter.new(@table.schema, @file.path) - chunk_size = 1024 - writer.write_table(@table, chunk_size) - writer.close - reader = Parquet::ArrowFileReader.new(@file.path) - @statistics = reader.metadata.get_row_group(0).get_column_chunk(0).statistics + Tempfile.create(["data", ".parquet"]) do |file| + @file = file + @table = build_table("int32" => build_int32_array([nil, -2, 9])) + writer = Parquet::ArrowFileWriter.new(@table.schema, @file.path) + chunk_size = 1024 + writer.write_table(@table, chunk_size) + writer.close + reader = Parquet::ArrowFileReader.new(@file.path) + begin + @statistics = + reader.metadata.get_row_group(0).get_column_chunk(0).statistics + yield + ensure + reader.unref + end + end end test("#min") do diff --git a/c_glib/test/parquet/test-int64-statistics.rb b/c_glib/test/parquet/test-int64-statistics.rb index 0a014573c1144..81fce8a0bbbbd 100644 --- a/c_glib/test/parquet/test-int64-statistics.rb +++ b/c_glib/test/parquet/test-int64-statistics.rb @@ -20,15 +20,23 @@ class TestParquetInt64Statistics < Test::Unit::TestCase def setup omit("Parquet is required") unless defined?(::Parquet) - @file = Tempfile.open(["data", ".parquet"]) - array = build_int64_array([nil, -(2 ** 32), 2 ** 32]) - @table = build_table("int64" => array) - writer = Parquet::ArrowFileWriter.new(@table.schema, @file.path) - chunk_size = 1024 - writer.write_table(@table, chunk_size) - writer.close - reader = Parquet::ArrowFileReader.new(@file.path) - @statistics = reader.metadata.get_row_group(0).get_column_chunk(0).statistics + Tempfile.create(["data", ".parquet"]) do |file| + @file = file + array = build_int64_array([nil, -(2 ** 32), 2 ** 32]) + @table = build_table("int64" => array) + writer = Parquet::ArrowFileWriter.new(@table.schema, @file.path) + chunk_size = 1024 + writer.write_table(@table, chunk_size) + writer.close + reader = Parquet::ArrowFileReader.new(@file.path) + begin + @statistics = + reader.metadata.get_row_group(0).get_column_chunk(0).statistics + yield + ensure + reader.unref + end + end end test("#min") do diff --git a/c_glib/test/parquet/test-row-group-metadata.rb b/c_glib/test/parquet/test-row-group-metadata.rb index e68cb9d11ee62..f238dd3b5774e 100644 --- a/c_glib/test/parquet/test-row-group-metadata.rb +++ b/c_glib/test/parquet/test-row-group-metadata.rb @@ -20,35 +20,46 @@ class TestParquetRowGroupMetadata < Test::Unit::TestCase def setup omit("Parquet is required") unless defined?(::Parquet) - @file = Tempfile.open(["data", ".parquet"]) - @string_array = build_string_array([nil, "hello"]) - fields = [ - Arrow::Field.new("int8", Arrow::Int8DataType.new), - Arrow::Field.new("boolean", Arrow::BooleanDataType.new), - ] - structs = [ - { - "int8" => -29, - "boolean" => true, - }, - nil, - ] - @struct_array = build_struct_array(fields, structs) - @table = build_table("string" => @string_array, - "struct" => @struct_array) - writer = Parquet::ArrowFileWriter.new(@table.schema, @file.path) - chunk_size = 1 - writer.write_table(@table, chunk_size) - writer.close - reader = Parquet::ArrowFileReader.new(@file.path) - @metadata = reader.metadata.get_row_group(0) + Tempfile.create(["data", ".parquet"]) do |file| + @file = file + @string_array = build_string_array([nil, "hello"]) + fields = [ + Arrow::Field.new("int8", Arrow::Int8DataType.new), + Arrow::Field.new("boolean", Arrow::BooleanDataType.new), + ] + structs = [ + { + "int8" => -29, + "boolean" => true, + }, + nil, + ] + @struct_array = build_struct_array(fields, structs) + @table = build_table("string" => @string_array, + "struct" => @struct_array) + writer = Parquet::ArrowFileWriter.new(@table.schema, @file.path) + chunk_size = 1 + writer.write_table(@table, chunk_size) + writer.close + reader = Parquet::ArrowFileReader.new(@file.path) + begin + @metadata = reader.metadata.get_row_group(0) + yield + ensure + reader.unref + end + end end test("#==") do reader = Parquet::ArrowFileReader.new(@file.path) - other_metadata = reader.metadata.get_row_group(0) - assert do - @metadata == other_metadata + begin + other_metadata = reader.metadata.get_row_group(0) + assert do + @metadata == other_metadata + end + ensure + reader.unref end end diff --git a/c_glib/test/parquet/test-statistics.rb b/c_glib/test/parquet/test-statistics.rb index 0367084c88a49..09a47ac255927 100644 --- a/c_glib/test/parquet/test-statistics.rb +++ b/c_glib/test/parquet/test-statistics.rb @@ -20,22 +20,34 @@ class TestParquetStatistics < Test::Unit::TestCase def setup omit("Parquet is required") unless defined?(::Parquet) - @file = Tempfile.open(["data", ".parquet"]) - @table = build_table("int32" => build_int32_array([nil, 2, 2, 9])) - writer = Parquet::ArrowFileWriter.new(@table.schema, @file.path) - chunk_size = 1024 - writer.write_table(@table, chunk_size) - writer.close - reader = Parquet::ArrowFileReader.new(@file.path) - @statistics = reader.metadata.get_row_group(0).get_column_chunk(0).statistics + Tempfile.create(["data", ".parquet"]) do |file| + @file = file + @table = build_table("int32" => build_int32_array([nil, 2, 2, 9])) + writer = Parquet::ArrowFileWriter.new(@table.schema, @file.path) + chunk_size = 1024 + writer.write_table(@table, chunk_size) + writer.close + reader = Parquet::ArrowFileReader.new(@file.path) + begin + @statistics = + reader.metadata.get_row_group(0).get_column_chunk(0).statistics + yield + ensure + reader.unref + end + end end test("#==") do reader = Parquet::ArrowFileReader.new(@file.path) - other_statistics = - reader.metadata.get_row_group(0).get_column_chunk(0).statistics - assert do - @statistics == other_statistics + begin + other_statistics = + reader.metadata.get_row_group(0).get_column_chunk(0).statistics + assert do + @statistics == other_statistics + end + ensure + reader.unref end end From 283f66f396401d6371b14a8ff66836de889bcfb0 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Sat, 25 May 2024 11:33:12 +0100 Subject: [PATCH 04/93] GH-41420: [R] Update NEWS.md for 16.1.0 (#41422) ### Rationale for this change Update NEWS.md ### What changes are included in this PR? News updates ### Are these changes tested? No ### Are there any user-facing changes? No * GitHub Issue: #41420 Authored-by: Nic Crane Signed-off-by: Nic Crane --- r/NEWS.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/r/NEWS.md b/r/NEWS.md index 47c4ac1571dad..dc89fa266e3ef 100644 --- a/r/NEWS.md +++ b/r/NEWS.md @@ -25,13 +25,22 @@ * `summarize()` supports more complex expressions, and correctly handles cases where column names are reused in expressions. * The `na_matches` argument to the `dplyr::*_join()` functions is now supported. This argument controls whether `NA` values are considered equal when joining. (#41358) -# arrow 16.0.0 +# arrow 16.1.0 + +## New features -# arrow 15.0.2 +* Streams can now be written to socket connections (#38897) +* The Arrow R package now can be built with older versions of the Arrow C++ library (back to 13.0.0) (#39738) -# arrow 15.0.1 +## Minor improvements and fixes + +* Dataset and table output printing now truncates schemas longer than 20 items long (#38916) +* Fixed pointer conversion to Python for latest reticulate to ensure data can be passed between Arrow and PyArrow (#39969) +* Check on macOS if we are using GNU libtool is and ensure we use macOS libtool instead (#40259) +* Fix an error where creating a bundled tarball with all dependencies was failing on Windows (@hutch3232, #40232) -# arrow 15.0.0 + +# arrow 15.0.1 ## New features From 1c9e393b73195840960dfb9eca8c0dc390be751a Mon Sep 17 00:00:00 2001 From: Adam Reeve Date: Sun, 26 May 2024 09:43:52 +1200 Subject: [PATCH 05/93] GH-41749: [GLib] Allow getting a RecordBatchReader from a Dataset or Scanner (#41750) ### Rationale for this change See #41749 ### What changes are included in this PR? Adds `to_reader` methods to `GADatasetDataset` and `GADatasetScanner`. ### Are these changes tested? Yes I've added new unit tests. ### Are there any user-facing changes? Yes this is a new feature. * GitHub Issue: #41749 Lead-authored-by: Adam Reeve Co-authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- c_glib/arrow-dataset-glib/dataset.cpp | 37 ++++++++++++++++++- c_glib/arrow-dataset-glib/dataset.h | 3 ++ c_glib/arrow-dataset-glib/scanner.cpp | 22 +++++++++++ c_glib/arrow-dataset-glib/scanner.h | 4 ++ .../test/dataset/test-file-system-dataset.rb | 24 ++++++++++-- c_glib/test/dataset/test-scanner.rb | 10 +++++ 6 files changed, 96 insertions(+), 4 deletions(-) diff --git a/c_glib/arrow-dataset-glib/dataset.cpp b/c_glib/arrow-dataset-glib/dataset.cpp index 704d6b589ee94..f84e4e3db380a 100644 --- a/c_glib/arrow-dataset-glib/dataset.cpp +++ b/c_glib/arrow-dataset-glib/dataset.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -152,12 +153,46 @@ gadataset_dataset_to_table(GADatasetDataset *dataset, GError **error) } auto arrow_scanner = *arrow_scanner_result; auto arrow_table_result = arrow_scanner->ToTable(); - if (!garrow::check(error, arrow_scanner_result, "[dataset][to-table]")) { + if (!garrow::check(error, arrow_table_result, "[dataset][to-table]")) { return NULL; } return garrow_table_new_raw(&(*arrow_table_result)); } +/** + * gadataset_dataset_to_record_batch_reader: + * @dataset: A #GADatasetDataset. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: (transfer full) (nullable): + * A #GArrowRecordBatchReader on success, %NULL on error. + * + * Since: 17.0.0 + */ +GArrowRecordBatchReader * +gadataset_dataset_to_record_batch_reader(GADatasetDataset *dataset, GError **error) +{ + auto arrow_dataset = gadataset_dataset_get_raw(dataset); + auto arrow_scanner_builder_result = arrow_dataset->NewScan(); + if (!garrow::check(error, + arrow_scanner_builder_result, + "[dataset][to-record-batch-reader]")) { + return nullptr; + } + auto arrow_scanner_builder = *arrow_scanner_builder_result; + auto arrow_scanner_result = arrow_scanner_builder->Finish(); + if (!garrow::check(error, arrow_scanner_result, "[dataset][to-record-batch-reader]")) { + return nullptr; + } + auto arrow_scanner = *arrow_scanner_result; + auto arrow_reader_result = arrow_scanner->ToRecordBatchReader(); + if (!garrow::check(error, arrow_reader_result, "[dataset][to-record-batch-reader]")) { + return nullptr; + } + auto sources = g_list_prepend(nullptr, dataset); + return garrow_record_batch_reader_new_raw(&(*arrow_reader_result), sources); +} + /** * gadataset_dataset_get_type_name: * @dataset: A #GADatasetDataset. diff --git a/c_glib/arrow-dataset-glib/dataset.h b/c_glib/arrow-dataset-glib/dataset.h index 657de330e6c49..5b957f0538a2a 100644 --- a/c_glib/arrow-dataset-glib/dataset.h +++ b/c_glib/arrow-dataset-glib/dataset.h @@ -34,6 +34,9 @@ gadataset_dataset_to_table(GADatasetDataset *dataset, GError **error); GADATASET_AVAILABLE_IN_5_0 gchar * gadataset_dataset_get_type_name(GADatasetDataset *dataset); +GADATASET_AVAILABLE_IN_17_0 +GArrowRecordBatchReader * +gadataset_dataset_to_record_batch_reader(GADatasetDataset *dataset, GError **error); #define GADATASET_TYPE_FILE_SYSTEM_DATASET_WRITE_OPTIONS \ (gadataset_file_system_dataset_write_options_get_type()) diff --git a/c_glib/arrow-dataset-glib/scanner.cpp b/c_glib/arrow-dataset-glib/scanner.cpp index 717532db9220f..28af1f16e5968 100644 --- a/c_glib/arrow-dataset-glib/scanner.cpp +++ b/c_glib/arrow-dataset-glib/scanner.cpp @@ -128,6 +128,28 @@ gadataset_scanner_to_table(GADatasetScanner *scanner, GError **error) } } +/** + * gadataset_scanner_to_record_batch_reader: + * @scanner: A #GADatasetScanner. + * @error: (nullable): Return location for a #GError or %NULL. + * + * Returns: (transfer full) (nullable): + * A #GArrowRecordBatchReader on success, %NULL on error. + * + * Since: 17.0.0 + */ +GArrowRecordBatchReader * +gadataset_scanner_to_record_batch_reader(GADatasetScanner *scanner, GError **error) +{ + auto arrow_scanner = gadataset_scanner_get_raw(scanner); + auto arrow_reader_result = arrow_scanner->ToRecordBatchReader(); + if (!garrow::check(error, arrow_reader_result, "[scanner][to-record-batch-reader]")) { + return nullptr; + } + auto sources = g_list_prepend(nullptr, scanner); + return garrow_record_batch_reader_new_raw(&(*arrow_reader_result), sources); +} + typedef struct GADatasetScannerBuilderPrivate_ { std::shared_ptr scanner_builder; diff --git a/c_glib/arrow-dataset-glib/scanner.h b/c_glib/arrow-dataset-glib/scanner.h index ad462391568a3..d92eca5ab8420 100644 --- a/c_glib/arrow-dataset-glib/scanner.h +++ b/c_glib/arrow-dataset-glib/scanner.h @@ -37,6 +37,10 @@ GADATASET_AVAILABLE_IN_5_0 GArrowTable * gadataset_scanner_to_table(GADatasetScanner *scanner, GError **error); +GADATASET_AVAILABLE_IN_17_0 +GArrowRecordBatchReader * +gadataset_scanner_to_record_batch_reader(GADatasetScanner *scanner, GError **error); + #define GADATASET_TYPE_SCANNER_BUILDER (gadataset_scanner_builder_get_type()) GADATASET_AVAILABLE_IN_5_0 G_DECLARE_DERIVABLE_TYPE( diff --git a/c_glib/test/dataset/test-file-system-dataset.rb b/c_glib/test/dataset/test-file-system-dataset.rb index 0e856b678f860..96deedf6b4eb0 100644 --- a/c_glib/test/dataset/test-file-system-dataset.rb +++ b/c_glib/test/dataset/test-file-system-dataset.rb @@ -56,6 +56,22 @@ def test_partitioning end def test_read_write + dataset, expected_table = create_dataset + assert_equal(expected_table, dataset.to_table) + end + + def test_to_record_batch_reader + dataset, expected_table = create_dataset + reader = dataset.to_record_batch_reader + begin + assert_equal(expected_table, reader.read_all) + ensure + # Unref to ensure the reader closes files and we can delete the temp directory + reader.unref + end + end + + def create_dataset table = build_table(label: build_string_array(["a", "a", "b", "c"]), count: build_int32_array([1, 10, 2, 3])) table_reader = Arrow::TableBatchReader.new(table) @@ -73,7 +89,8 @@ def test_read_write end @factory.partition_base_dir = @dir dataset = @factory.finish - assert_equal(build_table(count: [ + + expected_table = build_table(count: [ build_int32_array([1, 10]), build_int32_array([2]), build_int32_array([3]), @@ -82,7 +99,8 @@ def test_read_write build_string_array(["a", "a"]), build_string_array(["b"]), build_string_array(["c"]), - ]), - dataset.to_table) + ]) + + return dataset, expected_table end end diff --git a/c_glib/test/dataset/test-scanner.rb b/c_glib/test/dataset/test-scanner.rb index f7702d4905fb6..5dc31eefc5f4c 100644 --- a/c_glib/test/dataset/test-scanner.rb +++ b/c_glib/test/dataset/test-scanner.rb @@ -45,4 +45,14 @@ def setup def test_to_table assert_equal(@table, @scanner.to_table) end + + def test_to_record_batch_reader + reader = @scanner.to_record_batch_reader + begin + assert_equal(@table, reader.read_all) + ensure + # Unref to ensure the reader closes files and we can delete the temp directory + reader.unref + end + end end From ff9921ffa89585be69ae85674bb365d03cb22ba4 Mon Sep 17 00:00:00 2001 From: h-vetinari Date: Mon, 27 May 2024 18:59:52 +1100 Subject: [PATCH 06/93] GH-41755: [C++][ORC] Ensure setting detected ORC version (#41767) `FindorcAlt.cmake` doesn't set `orcAlt_VERSION` when it finds ORC by `find_library()`/`find_path()`. If `orcAlt_VERSION` isn't set, ORC version detection by caller is failed. `cpp/src/arrow/adapters/orc/adapter.cc` uses detected ORC version. If detected ORC version isn't correct, needless time zone database check is used. Deployed in conda-forge through https://github.com/conda-forge/arrow-cpp-feedstock/pull/1424 and confirmed as working in https://github.com/conda-forge/pyarrow-feedstock/pull/122 * GitHub Issue: #41755 Authored-by: H. Vetinari Signed-off-by: Sutou Kouhei --- cpp/cmake_modules/FindorcAlt.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/cmake_modules/FindorcAlt.cmake b/cpp/cmake_modules/FindorcAlt.cmake index 289416678ad39..ce8cd11b4c3f0 100644 --- a/cpp/cmake_modules/FindorcAlt.cmake +++ b/cpp/cmake_modules/FindorcAlt.cmake @@ -71,4 +71,5 @@ if(orcAlt_FOUND) PROPERTIES IMPORTED_LOCATION "${ORC_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${ORC_INCLUDE_DIR}") endif() + set(orcAlt_VERSION ${ORC_VERSION}) endif() From f8fe2ae3279f71cb2256024a9ab5d70dd9432f89 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 May 2024 08:44:56 +0900 Subject: [PATCH 07/93] MINOR: [Go] Bump github.com/goccy/go-json from 0.10.2 to 0.10.3 in /go (#41850) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/goccy/go-json](https://github.com/goccy/go-json) from 0.10.2 to 0.10.3.
Release notes

Sourced from github.com/goccy/go-json's releases.

0.10.3

What's Changed

New Contributors

Full Changelog: https://github.com/goccy/go-json/compare/v0.10.2...v0.10.3

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/goccy/go-json&package-manager=go_modules&previous-version=0.10.2&new-version=0.10.3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- go/go.mod | 2 +- go/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/go.mod b/go/go.mod index e846c61033f47..9c70544539b16 100644 --- a/go/go.mod +++ b/go/go.mod @@ -23,7 +23,7 @@ require ( github.com/andybalholm/brotli v1.1.0 github.com/apache/thrift v0.20.0 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 - github.com/goccy/go-json v0.10.2 + github.com/goccy/go-json v0.10.3 github.com/golang/snappy v0.0.4 github.com/google/flatbuffers v24.3.25+incompatible github.com/klauspost/asmfmt v1.3.2 diff --git a/go/go.sum b/go/go.sum index 6bceb4e5877ca..9e11041c333ac 100644 --- a/go/go.sum +++ b/go/go.sum @@ -26,8 +26,8 @@ github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD87 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= -github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= -github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.11.0 h1:n7Z+zx8S9f9KgzG6KtQKf+kwqXZlLNR2F6018Dgau54= github.com/goccy/go-yaml v1.11.0/go.mod h1:H+mJrWtjPTJAHvRbV09MCK9xYwODM+wRTVFFTWckfng= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= From ef3d4670e70a40f46e6acdd7041b5ee7edb16a42 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 16:58:31 -0700 Subject: [PATCH 08/93] MINOR: [C#] Bump xunit from 2.8.0 to 2.8.1 in /csharp (#41842) Bumps [xunit](https://github.com/xunit/xunit) from 2.8.0 to 2.8.1.
Commits
  • ba2ae9b v2.8.1
  • 151b8d0 Use 'dotnet format' instead of 'dotnet dotnet-format'
  • be6db6f #2931: Tighten up types to prevent accidentically calling AddOrGet with a Con...
  • f466d81 #2927: Misleading error message when class used in IClassFixture<> throws exc...
  • 1911ea7 Missed unit test updates
  • f497d65 Cannot use full assembly path as dictionary key for execution options lookup,...
  • c2f2d47 Add ability to provide live output messages from running tests
  • 013093b Polyfill in StringSyntaxAttribute
  • e1e4c2e #2719: Class with custom Fact with throwing Skip should fail appropriately
  • 8b0b13c Clarify naming
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=xunit&package-manager=nuget&previous-version=2.8.0&new-version=2.8.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Curt Hagenlocher --- .../Apache.Arrow.Compression.Tests.csproj | 2 +- .../Apache.Arrow.Flight.Sql.Tests.csproj | 2 +- .../Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj | 2 +- csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/csharp/test/Apache.Arrow.Compression.Tests/Apache.Arrow.Compression.Tests.csproj b/csharp/test/Apache.Arrow.Compression.Tests/Apache.Arrow.Compression.Tests.csproj index 2b1720561004e..a02dc383fb4e8 100644 --- a/csharp/test/Apache.Arrow.Compression.Tests/Apache.Arrow.Compression.Tests.csproj +++ b/csharp/test/Apache.Arrow.Compression.Tests/Apache.Arrow.Compression.Tests.csproj @@ -8,7 +8,7 @@ - + diff --git a/csharp/test/Apache.Arrow.Flight.Sql.Tests/Apache.Arrow.Flight.Sql.Tests.csproj b/csharp/test/Apache.Arrow.Flight.Sql.Tests/Apache.Arrow.Flight.Sql.Tests.csproj index c8fb40f2d6702..50e5cea7a1dd0 100644 --- a/csharp/test/Apache.Arrow.Flight.Sql.Tests/Apache.Arrow.Flight.Sql.Tests.csproj +++ b/csharp/test/Apache.Arrow.Flight.Sql.Tests/Apache.Arrow.Flight.Sql.Tests.csproj @@ -7,7 +7,7 @@ - + diff --git a/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj b/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj index ba60451f25f68..7b4d063763fde 100644 --- a/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj +++ b/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj @@ -7,7 +7,7 @@ - + diff --git a/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj b/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj index 90b498d4e9b03..776f37db3a6d7 100644 --- a/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj +++ b/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj @@ -17,7 +17,7 @@ - + all runtime; build; native; contentfiles; analyzers From 85a2ac9168ac5f003e0a38c9458ed10f3534abe6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 16:59:51 -0700 Subject: [PATCH 09/93] MINOR: [C#] Bump Grpc.AspNetCore, Grpc.AspNetCore.Server, System.Runtime.CompilerServices.Unsafe and Grpc.Net.Client in /csharp (#41843) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [Grpc.AspNetCore](https://github.com/grpc/grpc-dotnet), [Grpc.AspNetCore.Server](https://github.com/grpc/grpc-dotnet), [System.Runtime.CompilerServices.Unsafe](https://github.com/dotnet/runtime) and [Grpc.Net.Client](https://github.com/grpc/grpc-dotnet). These dependencies needed to be updated together. Updates `Grpc.AspNetCore` from 2.62.0 to 2.63.0
Release notes

Sourced from Grpc.AspNetCore's releases.

Release v2.63.0

What's Changed

New Contributors

Full Changelog: https://github.com/grpc/grpc-dotnet/compare/v2.62.0...v2.63.0

Release v2.63.0-pre1

What's Changed

New Contributors

... (truncated)

Commits
  • 6f1271f bump to stable release (#2440)
  • ac75fe3 build: configure rollForward in global.json (#2442)
  • 48aeb80 Bump 2.63.x branch to 2.63.0-pre1 (#2430)
  • 989420e Update Grpc.Tools to 2.63 (#2429)
  • c80f459 Don't capture async locals in resolver (#2426)
  • 63914f2 Add semaphore to limit subchannel connect to prevent race conditions (#2422)
  • 8199f66 Fix HTTP/3 test errors on .NET 6 (#2423)
  • 2d9df58 Fix memory leak when using call context propagation with cancellation token (...
  • c9c902c Enable multiple connections with WinHttpHandler by default (#2416)
  • 2a36215 Fix ObjectDisposedException message (#2415)
  • Additional commits viewable in compare view

Updates `Grpc.AspNetCore.Server` from 2.62.0 to 2.63.0
Release notes

Sourced from Grpc.AspNetCore.Server's releases.

Release v2.63.0

What's Changed

New Contributors

Full Changelog: https://github.com/grpc/grpc-dotnet/compare/v2.62.0...v2.63.0

Release v2.63.0-pre1

What's Changed

New Contributors

... (truncated)

Commits
  • 6f1271f bump to stable release (#2440)
  • ac75fe3 build: configure rollForward in global.json (#2442)
  • 48aeb80 Bump 2.63.x branch to 2.63.0-pre1 (#2430)
  • 989420e Update Grpc.Tools to 2.63 (#2429)
  • c80f459 Don't capture async locals in resolver (#2426)
  • 63914f2 Add semaphore to limit subchannel connect to prevent race conditions (#2422)
  • 8199f66 Fix HTTP/3 test errors on .NET 6 (#2423)
  • 2d9df58 Fix memory leak when using call context propagation with cancellation token (...
  • c9c902c Enable multiple connections with WinHttpHandler by default (#2416)
  • 2a36215 Fix ObjectDisposedException message (#2415)
  • Additional commits viewable in compare view

Updates `System.Runtime.CompilerServices.Unsafe` from 4.7.1 to 6.0.0
Release notes

Sourced from System.Runtime.CompilerServices.Unsafe's releases.

.NET 6.0

Release

.NET 6.0 RC 2

Release

.NET 6.0 RC 1

Release

.NET 6.0 Preview 7

Release

.NET 6.0 Preview 6

Release

.NET 6.0 Preview 5

Release

.NET 6.0 Preview 4

Release

.NET 6.0 Preview 3

Release

.NET 6.0 Preview 2

Release

.NET 6.0 Preview 1

Release

.NET 5.0.17

Release

.NET 5 is now out of support. We recommend using .NET 6.

.NET 5.0.16

Release

.NET 5.0.15

Release

.NET 5.0.14

Release

.NET 5.0.13

Release

.NET 5.0.11

Release

... (truncated)

Commits

Updates `Grpc.Net.Client` from 2.59.0 to 2.63.0
Release notes

Sourced from Grpc.Net.Client's releases.

Release v2.63.0

What's Changed

New Contributors

Full Changelog: https://github.com/grpc/grpc-dotnet/compare/v2.62.0...v2.63.0

Release v2.63.0-pre1

What's Changed

New Contributors

... (truncated)

Commits
  • 6f1271f bump to stable release (#2440)
  • ac75fe3 build: configure rollForward in global.json (#2442)
  • 48aeb80 Bump 2.63.x branch to 2.63.0-pre1 (#2430)
  • 989420e Update Grpc.Tools to 2.63 (#2429)
  • c80f459 Don't capture async locals in resolver (#2426)
  • 63914f2 Add semaphore to limit subchannel connect to prevent race conditions (#2422)
  • 8199f66 Fix HTTP/3 test errors on .NET 6 (#2423)
  • 2d9df58 Fix memory leak when using call context propagation with cancellation token (...
  • c9c902c Enable multiple connections with WinHttpHandler by default (#2416)
  • 2a36215 Fix ObjectDisposedException message (#2415)
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Curt Hagenlocher --- .../Apache.Arrow.Flight.TestWeb.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/test/Apache.Arrow.Flight.TestWeb/Apache.Arrow.Flight.TestWeb.csproj b/csharp/test/Apache.Arrow.Flight.TestWeb/Apache.Arrow.Flight.TestWeb.csproj index bd6425e7ed99b..789fb9569edba 100644 --- a/csharp/test/Apache.Arrow.Flight.TestWeb/Apache.Arrow.Flight.TestWeb.csproj +++ b/csharp/test/Apache.Arrow.Flight.TestWeb/Apache.Arrow.Flight.TestWeb.csproj @@ -5,7 +5,7 @@ - + From 95c8f0a031bb5659feb42f526fb5710282165de9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 17:01:07 -0700 Subject: [PATCH 10/93] MINOR: [C#] Bump Grpc.Tools from 2.63.0 to 2.64.0 in /csharp (#41844) Bumps [Grpc.Tools](https://github.com/grpc/grpc) from 2.63.0 to 2.64.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=Grpc.Tools&package-manager=nuget&previous-version=2.63.0&new-version=2.64.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Curt Hagenlocher --- .../src/Apache.Arrow.Flight.Sql/Apache.Arrow.Flight.Sql.csproj | 2 +- csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/csharp/src/Apache.Arrow.Flight.Sql/Apache.Arrow.Flight.Sql.csproj b/csharp/src/Apache.Arrow.Flight.Sql/Apache.Arrow.Flight.Sql.csproj index 7314b8207fef6..ee6d42c8d17fc 100644 --- a/csharp/src/Apache.Arrow.Flight.Sql/Apache.Arrow.Flight.Sql.csproj +++ b/csharp/src/Apache.Arrow.Flight.Sql/Apache.Arrow.Flight.Sql.csproj @@ -5,7 +5,7 @@ - + diff --git a/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj b/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj index 780da3ad39081..d68cdfae30010 100644 --- a/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj +++ b/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj @@ -7,7 +7,7 @@ - + From aa00b8b7e5cc82316ad3c2a3d625784536cd9de9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 17:02:33 -0700 Subject: [PATCH 11/93] MINOR: [C#] Bump Grpc.AspNetCore.Server from 2.62.0 to 2.63.0 in /csharp (#41846) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [Grpc.AspNetCore.Server](https://github.com/grpc/grpc-dotnet) from 2.62.0 to 2.63.0.
Release notes

Sourced from Grpc.AspNetCore.Server's releases.

Release v2.63.0

What's Changed

New Contributors

Full Changelog: https://github.com/grpc/grpc-dotnet/compare/v2.62.0...v2.63.0

Release v2.63.0-pre1

What's Changed

New Contributors

... (truncated)

Commits
  • 6f1271f bump to stable release (#2440)
  • ac75fe3 build: configure rollForward in global.json (#2442)
  • 48aeb80 Bump 2.63.x branch to 2.63.0-pre1 (#2430)
  • 989420e Update Grpc.Tools to 2.63 (#2429)
  • c80f459 Don't capture async locals in resolver (#2426)
  • 63914f2 Add semaphore to limit subchannel connect to prevent race conditions (#2422)
  • 8199f66 Fix HTTP/3 test errors on .NET 6 (#2423)
  • 2d9df58 Fix memory leak when using call context propagation with cancellation token (...
  • c9c902c Enable multiple connections with WinHttpHandler by default (#2416)
  • 2a36215 Fix ObjectDisposedException message (#2415)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=Grpc.AspNetCore.Server&package-manager=nuget&previous-version=2.62.0&new-version=2.63.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Curt Hagenlocher --- .../Apache.Arrow.Flight.AspNetCore.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/src/Apache.Arrow.Flight.AspNetCore/Apache.Arrow.Flight.AspNetCore.csproj b/csharp/src/Apache.Arrow.Flight.AspNetCore/Apache.Arrow.Flight.AspNetCore.csproj index 55497203a12be..2dd1d9d8f98e2 100644 --- a/csharp/src/Apache.Arrow.Flight.AspNetCore/Apache.Arrow.Flight.AspNetCore.csproj +++ b/csharp/src/Apache.Arrow.Flight.AspNetCore/Apache.Arrow.Flight.AspNetCore.csproj @@ -5,7 +5,7 @@ - + From 4b49f50ba7ff649b6b83bb4b849b4fc33eb77adb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 17:03:31 -0700 Subject: [PATCH 12/93] MINOR: [C#] Bump Google.Protobuf from 3.26.1 to 3.27.0 in /csharp (#41847) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [//]: # (dependabot-start) ⚠️ **Dependabot is rebasing this PR** ⚠️ Rebasing might not happen immediately, so don't worry if this takes some time. Note: if you make any changes to this PR yourself, they will take precedence over the rebase. --- [//]: # (dependabot-end) Bumps [Google.Protobuf](https://github.com/protocolbuffers/protobuf) from 3.26.1 to 3.27.0.
Commits
  • a978b75 Updating version.json and repo version numbers to: 27.0
  • f396506 MODULE.bazel fixes for protobuf BCR release. (#16927)
  • 4baa11f Merge pull request #16906 from protocolbuffers/editions-27
  • 4483c6b Lazily resolve features for proto2 and proto3 for compatibility with old open...
  • f8a4a68 Fix whitespace merge issue in gencode
  • c3417f5 Regenerate stale files
  • fea6847 Future-proof edition 2023 against feature value support windows.
  • d2da463 Mark deleted fields in descriptor.proto reserved
  • 0a05aa8 Merge pull request #16875 from protocolbuffers/ban-recursive-features
  • 8c5f3a7 Prohibit using features in the same file they're defined in.
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=Google.Protobuf&package-manager=nuget&previous-version=3.26.1&new-version=3.27.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Curt Hagenlocher --- csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj b/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj index d68cdfae30010..d3892a6de16ee 100644 --- a/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj +++ b/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj @@ -5,7 +5,7 @@ - + From cf18331480d7f086a418bac0fa569a29d3fb7664 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 20:41:18 -0700 Subject: [PATCH 13/93] MINOR: [C#] Bump Grpc.Net.Client and System.Runtime.CompilerServices.Unsafe in /csharp (#41845) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [Grpc.Net.Client](https://github.com/grpc/grpc-dotnet) and [System.Runtime.CompilerServices.Unsafe](https://github.com/dotnet/runtime). These dependencies needed to be updated together. Updates `Grpc.Net.Client` from 2.59.0 to 2.63.0
Release notes

Sourced from Grpc.Net.Client's releases.

Release v2.63.0

What's Changed

New Contributors

Full Changelog: https://github.com/grpc/grpc-dotnet/compare/v2.62.0...v2.63.0

Release v2.63.0-pre1

What's Changed

New Contributors

... (truncated)

Commits
  • 6f1271f bump to stable release (#2440)
  • ac75fe3 build: configure rollForward in global.json (#2442)
  • 48aeb80 Bump 2.63.x branch to 2.63.0-pre1 (#2430)
  • 989420e Update Grpc.Tools to 2.63 (#2429)
  • c80f459 Don't capture async locals in resolver (#2426)
  • 63914f2 Add semaphore to limit subchannel connect to prevent race conditions (#2422)
  • 8199f66 Fix HTTP/3 test errors on .NET 6 (#2423)
  • 2d9df58 Fix memory leak when using call context propagation with cancellation token (...
  • c9c902c Enable multiple connections with WinHttpHandler by default (#2416)
  • 2a36215 Fix ObjectDisposedException message (#2415)
  • Additional commits viewable in compare view

Updates `System.Runtime.CompilerServices.Unsafe` from 4.7.1 to 6.0.0
Release notes

Sourced from System.Runtime.CompilerServices.Unsafe's releases.

.NET 6.0

Release

.NET 6.0 RC 2

Release

.NET 6.0 RC 1

Release

.NET 6.0 Preview 7

Release

.NET 6.0 Preview 6

Release

.NET 6.0 Preview 5

Release

.NET 6.0 Preview 4

Release

.NET 6.0 Preview 3

Release

.NET 6.0 Preview 2

Release

.NET 6.0 Preview 1

Release

.NET 5.0.17

Release

.NET 5 is now out of support. We recommend using .NET 6.

.NET 5.0.16

Release

.NET 5.0.15

Release

.NET 5.0.14

Release

.NET 5.0.13

Release

.NET 5.0.11

Release

... (truncated)

Commits

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Curt Hagenlocher --- csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj b/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj index d3892a6de16ee..21b0df349e2d8 100644 --- a/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj +++ b/csharp/src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj @@ -6,7 +6,7 @@ - + From a34d995af8510e24654ccdb347343bd793ac1fed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 20:42:00 -0700 Subject: [PATCH 14/93] MINOR: [C#] Bump Microsoft.NET.Test.Sdk from 17.9.0 to 17.10.0 in /csharp (#41848) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [Microsoft.NET.Test.Sdk](https://github.com/microsoft/vstest) from 17.9.0 to 17.10.0.
Release notes

Sourced from Microsoft.NET.Test.Sdk's releases.

v17.10.0

What's Changed

And many infrastructure related changes and updates.

New Contributors

Full Changelog: https://github.com/microsoft/vstest/compare/v17.9.0...v17.10.0

v17.10.0-release-24177-07

What's Changed

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=Microsoft.NET.Test.Sdk&package-manager=nuget&previous-version=17.9.0&new-version=17.10.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Curt Hagenlocher --- .../Apache.Arrow.Compression.Tests.csproj | 2 +- .../Apache.Arrow.Flight.Sql.Tests.csproj | 2 +- .../Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj | 2 +- csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/csharp/test/Apache.Arrow.Compression.Tests/Apache.Arrow.Compression.Tests.csproj b/csharp/test/Apache.Arrow.Compression.Tests/Apache.Arrow.Compression.Tests.csproj index a02dc383fb4e8..f4780c0dad194 100644 --- a/csharp/test/Apache.Arrow.Compression.Tests/Apache.Arrow.Compression.Tests.csproj +++ b/csharp/test/Apache.Arrow.Compression.Tests/Apache.Arrow.Compression.Tests.csproj @@ -7,7 +7,7 @@ - + diff --git a/csharp/test/Apache.Arrow.Flight.Sql.Tests/Apache.Arrow.Flight.Sql.Tests.csproj b/csharp/test/Apache.Arrow.Flight.Sql.Tests/Apache.Arrow.Flight.Sql.Tests.csproj index 50e5cea7a1dd0..d3caaf9ca0fa8 100644 --- a/csharp/test/Apache.Arrow.Flight.Sql.Tests/Apache.Arrow.Flight.Sql.Tests.csproj +++ b/csharp/test/Apache.Arrow.Flight.Sql.Tests/Apache.Arrow.Flight.Sql.Tests.csproj @@ -6,7 +6,7 @@ - + diff --git a/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj b/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj index 7b4d063763fde..37726b85f8fe4 100644 --- a/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj +++ b/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj @@ -6,7 +6,7 @@ - + diff --git a/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj b/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj index 776f37db3a6d7..067d41a42524f 100644 --- a/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj +++ b/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj @@ -16,7 +16,7 @@ - + all From 5a667281efec2d5f8789fbadc2b27924ab5b103f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 20:42:54 -0700 Subject: [PATCH 15/93] MINOR: [C#] Bump xunit.runner.visualstudio from 2.8.0 to 2.8.1 in /csharp (#41849) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [//]: # (dependabot-start) ⚠️ **Dependabot is rebasing this PR** ⚠️ Rebasing might not happen immediately, so don't worry if this takes some time. Note: if you make any changes to this PR yourself, they will take precedence over the rebase. --- [//]: # (dependabot-end) Bumps [xunit.runner.visualstudio](https://github.com/xunit/visualstudio.xunit) from 2.8.0 to 2.8.1.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=xunit.runner.visualstudio&package-manager=nuget&previous-version=2.8.0&new-version=2.8.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Curt Hagenlocher --- .../Apache.Arrow.Compression.Tests.csproj | 2 +- .../Apache.Arrow.Flight.Sql.Tests.csproj | 2 +- .../Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj | 2 +- csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/csharp/test/Apache.Arrow.Compression.Tests/Apache.Arrow.Compression.Tests.csproj b/csharp/test/Apache.Arrow.Compression.Tests/Apache.Arrow.Compression.Tests.csproj index f4780c0dad194..bd97372d1021b 100644 --- a/csharp/test/Apache.Arrow.Compression.Tests/Apache.Arrow.Compression.Tests.csproj +++ b/csharp/test/Apache.Arrow.Compression.Tests/Apache.Arrow.Compression.Tests.csproj @@ -9,7 +9,7 @@ - + diff --git a/csharp/test/Apache.Arrow.Flight.Sql.Tests/Apache.Arrow.Flight.Sql.Tests.csproj b/csharp/test/Apache.Arrow.Flight.Sql.Tests/Apache.Arrow.Flight.Sql.Tests.csproj index d3caaf9ca0fa8..5a5a92ccd2c7f 100644 --- a/csharp/test/Apache.Arrow.Flight.Sql.Tests/Apache.Arrow.Flight.Sql.Tests.csproj +++ b/csharp/test/Apache.Arrow.Flight.Sql.Tests/Apache.Arrow.Flight.Sql.Tests.csproj @@ -8,7 +8,7 @@ - + diff --git a/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj b/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj index 37726b85f8fe4..132f17fa212a5 100644 --- a/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj +++ b/csharp/test/Apache.Arrow.Flight.Tests/Apache.Arrow.Flight.Tests.csproj @@ -8,7 +8,7 @@ - + diff --git a/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj b/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj index 067d41a42524f..a3290e3be14ee 100644 --- a/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj +++ b/csharp/test/Apache.Arrow.Tests/Apache.Arrow.Tests.csproj @@ -18,7 +18,7 @@ - + all runtime; build; native; contentfiles; analyzers From f63c9943d6b8cd2d40b0d3dab69de7b6e4abe6e2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 20:44:35 -0700 Subject: [PATCH 16/93] MINOR: [C#] Bump ZstdSharp.Port and System.Runtime.CompilerServices.Unsafe in /csharp (#41742) Bumps [ZstdSharp.Port](https://github.com/oleg-st/ZstdSharp) and [System.Runtime.CompilerServices.Unsafe](https://github.com/dotnet/runtime). These dependencies needed to be updated together. Updates `ZstdSharp.Port` from 0.8.0 to 0.8.1
Release notes

Sourced from ZstdSharp.Port's releases.

0.8.1

Better decompression speed in .NET 8 (~5%)

Commits

Updates `System.Runtime.CompilerServices.Unsafe` from 4.7.1 to 6.0.0
Release notes

Sourced from System.Runtime.CompilerServices.Unsafe's releases.

.NET 6.0

Release

.NET 6.0 RC 2

Release

.NET 6.0 RC 1

Release

.NET 6.0 Preview 7

Release

.NET 6.0 Preview 6

Release

.NET 6.0 Preview 5

Release

.NET 6.0 Preview 4

Release

.NET 6.0 Preview 3

Release

.NET 6.0 Preview 2

Release

.NET 6.0 Preview 1

Release

.NET 5.0.17

Release

.NET 5 is now out of support. We recommend using .NET 6.

.NET 5.0.16

Release

.NET 5.0.15

Release

.NET 5.0.14

Release

.NET 5.0.13

Release

.NET 5.0.11

Release

... (truncated)

Commits

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Curt Hagenlocher --- .../Apache.Arrow.Compression/Apache.Arrow.Compression.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/src/Apache.Arrow.Compression/Apache.Arrow.Compression.csproj b/csharp/src/Apache.Arrow.Compression/Apache.Arrow.Compression.csproj index c34d880f90060..b8f69672cbc7c 100644 --- a/csharp/src/Apache.Arrow.Compression/Apache.Arrow.Compression.csproj +++ b/csharp/src/Apache.Arrow.Compression/Apache.Arrow.Compression.csproj @@ -13,7 +13,7 @@ - + From f904928054fad89360d83015db5c23ac1ef86d05 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 28 May 2024 13:17:35 +0900 Subject: [PATCH 17/93] GH-41784: [Packaging][RPM] Use SO version for -libs package name (#41838) ### Rationale for this change We should use `arrow${SO_VERSION}-libs` not `arrow${MAJOR_VERSION}-libs` to co-exist newer versions and older versions. ### What changes are included in this PR? Use SO version not major version. ### Are these changes tested? Yes. ### Are there any user-facing changes? Yes. * GitHub Issue: #41784 Authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- .../apache-arrow/yum/arrow.spec.in | 148 +++++++++--------- 1 file changed, 75 insertions(+), 73 deletions(-) diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index c6148e9260586..d5e6c3a332eb3 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -27,7 +27,9 @@ %define is_centos_7 (%{_rhel} == 7 && !%{is_amazon_linux}) -%define major_version %(echo @VERSION@ | grep -o '^[0-9]*') +%define major_version %(echo @VERSION@ | cut -d. -f 1) +%define minor_version %(echo @VERSION@ | cut -d. -f 2) +%define so_version %(expr %{major_version} '*' 100 + %{minor_version}) %define boost_version %( \ if [ %{_rhel} -eq 7 ]; then \ @@ -239,7 +241,7 @@ cd cpp rm -rf %{buildroot}%{_docdir}/arrow/ cd - -%package -n %{name}%{major_version}-libs +%package -n %{name}%{so_version}-libs Summary: Runtime libraries for Apache Arrow C++ License: Apache-2.0 %if %{have_lz4_libs} @@ -248,10 +250,10 @@ Requires: lz4-libs %{lz4_requirement} Requires: lz4 %{lz4_requirement} %endif -%description -n %{name}%{major_version}-libs +%description -n %{name}%{so_version}-libs This package contains the libraries for Apache Arrow C++. -%files -n %{name}%{major_version}-libs +%files -n %{name}%{so_version}-libs %defattr(-,root,root,-) %doc README.md %license LICENSE.txt NOTICE.txt @@ -260,7 +262,7 @@ This package contains the libraries for Apache Arrow C++. %package tools Summary: Tools for Apache Arrow C++ License: Apache-2.0 -Requires: %{name}%{major_version}-libs = %{version}-%{release} +Requires: %{name}%{so_version}-libs = %{version}-%{release} %description tools Tools for Apache Arrow C++. @@ -274,7 +276,7 @@ Tools for Apache Arrow C++. %package devel Summary: Libraries and header files for Apache Arrow C++ License: Apache-2.0 -Requires: %{name}%{major_version}-libs = %{version}-%{release} +Requires: %{name}%{so_version}-libs = %{version}-%{release} Requires: brotli-devel Requires: bzip2-devel Requires: curl-devel @@ -322,15 +324,15 @@ Libraries and header files for Apache Arrow C++. %{_libdir}/pkgconfig/arrow-orc.pc %{_libdir}/pkgconfig/arrow.pc -%package -n %{name}%{major_version}-acero-libs +%package -n %{name}%{so_version}-acero-libs Summary: C++ library to execute a query in streaming License: Apache-2.0 -Requires: %{name}%{major_version}-libs = %{version}-%{release} +Requires: %{name}%{so_version}-libs = %{version}-%{release} -%description -n %{name}%{major_version}-acero-libs +%description -n %{name}%{so_version}-acero-libs This package contains the libraries for Apache Arrow Acero. -%files -n %{name}%{major_version}-acero-libs +%files -n %{name}%{so_version}-acero-libs %defattr(-,root,root,-) %doc README.md %license LICENSE.txt NOTICE.txt @@ -339,7 +341,7 @@ This package contains the libraries for Apache Arrow Acero. %package acero-devel Summary: Libraries and header files for Apache Arrow Acero License: Apache-2.0 -Requires: %{name}%{major_version}-acero-libs = %{version}-%{release} +Requires: %{name}%{so_version}-acero-libs = %{version}-%{release} Requires: %{name}-devel = %{version}-%{release} %description acero-devel @@ -355,16 +357,16 @@ Libraries and header files for Apache Arrow Acero %{_libdir}/libarrow_acero.so %{_libdir}/pkgconfig/arrow-acero.pc -%package -n %{name}%{major_version}-dataset-libs +%package -n %{name}%{so_version}-dataset-libs Summary: C++ library to read and write semantic datasets stored in different locations and formats License: Apache-2.0 -Requires: %{name}%{major_version}-acero-libs = %{version}-%{release} -Requires: parquet%{major_version}-libs = %{version}-%{release} +Requires: %{name}%{so_version}-acero-libs = %{version}-%{release} +Requires: parquet%{so_version}-libs = %{version}-%{release} -%description -n %{name}%{major_version}-dataset-libs +%description -n %{name}%{so_version}-dataset-libs This package contains the libraries for Apache Arrow dataset. -%files -n %{name}%{major_version}-dataset-libs +%files -n %{name}%{so_version}-dataset-libs %defattr(-,root,root,-) %doc README.md %license LICENSE.txt NOTICE.txt @@ -373,7 +375,7 @@ This package contains the libraries for Apache Arrow dataset. %package dataset-devel Summary: Libraries and header files for Apache Arrow dataset. License: Apache-2.0 -Requires: %{name}%{major_version}-dataset-libs = %{version}-%{release} +Requires: %{name}%{so_version}-dataset-libs = %{version}-%{release} Requires: %{name}-acero-devel = %{version}-%{release} Requires: parquet-devel = %{version}-%{release} @@ -391,15 +393,15 @@ Libraries and header files for Apache Arrow dataset. %{_libdir}/pkgconfig/arrow-dataset.pc %if %{use_flight} -%package -n %{name}%{major_version}-flight-libs +%package -n %{name}%{so_version}-flight-libs Summary: C++ library for fast data transport. License: Apache-2.0 -Requires: %{name}%{major_version}-libs = %{version}-%{release} +Requires: %{name}%{so_version}-libs = %{version}-%{release} -%description -n %{name}%{major_version}-flight-libs +%description -n %{name}%{so_version}-flight-libs This package contains the libraries for Apache Arrow Flight. -%files -n %{name}%{major_version}-flight-libs +%files -n %{name}%{so_version}-flight-libs %defattr(-,root,root,-) %doc README.md %license LICENSE.txt NOTICE.txt @@ -408,7 +410,7 @@ This package contains the libraries for Apache Arrow Flight. %package flight-devel Summary: Libraries and header files for Apache Arrow Flight. License: Apache-2.0 -Requires: %{name}%{major_version}-flight-libs = %{version}-%{release} +Requires: %{name}%{so_version}-flight-libs = %{version}-%{release} Requires: %{name}-devel = %{version}-%{release} Requires: c-ares-devel %if %{have_grpc} @@ -430,15 +432,15 @@ Libraries and header files for Apache Arrow Flight. %{_libdir}/libarrow_flight.so %{_libdir}/pkgconfig/arrow-flight.pc -%package -n %{name}%{major_version}-flight-sql-libs +%package -n %{name}%{so_version}-flight-sql-libs Summary: C++ library for interacting with SQL databases. License: Apache-2.0 -Requires: %{name}%{major_version}-flight-libs = %{version}-%{release} +Requires: %{name}%{so_version}-flight-libs = %{version}-%{release} -%description -n %{name}%{major_version}-flight-sql-libs +%description -n %{name}%{so_version}-flight-sql-libs This package contains the libraries for Apache Arrow Flight SQL. -%files -n %{name}%{major_version}-flight-sql-libs +%files -n %{name}%{so_version}-flight-sql-libs %defattr(-,root,root,-) %doc README.md %license LICENSE.txt NOTICE.txt @@ -447,7 +449,7 @@ This package contains the libraries for Apache Arrow Flight SQL. %package flight-sql-devel Summary: Libraries and header files for Apache Arrow Flight SQL. License: Apache-2.0 -Requires: %{name}%{major_version}-flight-sql-libs = %{version}-%{release} +Requires: %{name}%{so_version}-flight-sql-libs = %{version}-%{release} Requires: %{name}-devel = %{version}-%{release} %description flight-sql-devel @@ -465,15 +467,15 @@ Libraries and header files for Apache Arrow Flight SQL. %endif %if %{use_gandiva} -%package -n gandiva%{major_version}-libs +%package -n gandiva%{so_version}-libs Summary: C++ library for compiling and evaluating expressions on Apache Arrow data. License: Apache-2.0 -Requires: %{name}%{major_version}-libs = %{version}-%{release} +Requires: %{name}%{so_version}-libs = %{version}-%{release} -%description -n gandiva%{major_version}-libs +%description -n gandiva%{so_version}-libs This package contains the libraries for Gandiva. -%files -n gandiva%{major_version}-libs +%files -n gandiva%{so_version}-libs %defattr(-,root,root,-) %doc README.md %license LICENSE.txt NOTICE.txt @@ -483,7 +485,7 @@ This package contains the libraries for Gandiva. Summary: Libraries and header files for Gandiva. License: Apache-2.0 Requires: %{name}-devel = %{version}-%{release} -Requires: gandiva%{major_version}-libs = %{version}-%{release} +Requires: gandiva%{so_version}-libs = %{version}-%{release} Requires: llvm-devel %description -n gandiva-devel @@ -500,15 +502,15 @@ Libraries and header files for Gandiva. %{_libdir}/pkgconfig/gandiva.pc %endif -%package -n parquet%{major_version}-libs +%package -n parquet%{so_version}-libs Summary: Runtime libraries for Apache Parquet C++ License: Apache-2.0 -Requires: %{name}%{major_version}-libs = %{version}-%{release} +Requires: %{name}%{so_version}-libs = %{version}-%{release} -%description -n parquet%{major_version}-libs +%description -n parquet%{so_version}-libs This package contains the libraries for Apache Parquet C++. -%files -n parquet%{major_version}-libs +%files -n parquet%{so_version}-libs %defattr(-,root,root,-) %doc README.md %license LICENSE.txt NOTICE.txt @@ -517,7 +519,7 @@ This package contains the libraries for Apache Parquet C++. %package -n parquet-tools Summary: Tools for Apache Parquet C++ License: Apache-2.0 -Requires: parquet%{major_version}-libs = %{version}-%{release} +Requires: parquet%{so_version}-libs = %{version}-%{release} %description -n parquet-tools Tools for Apache Parquet C++. @@ -532,7 +534,7 @@ Tools for Apache Parquet C++. Summary: Libraries and header files for Apache Parquet C++ License: Apache-2.0 Requires: %{name}-devel = %{version}-%{release} -Requires: parquet%{major_version}-libs = %{version}-%{release} +Requires: parquet%{so_version}-libs = %{version}-%{release} Requires: zlib-devel %description -n parquet-devel @@ -548,15 +550,15 @@ Libraries and header files for Apache Parquet C++. %{_libdir}/libparquet.so %{_libdir}/pkgconfig/parquet*.pc -%package -n %{name}%{major_version}-glib-libs +%package -n %{name}%{so_version}-glib-libs Summary: Runtime libraries for Apache Arrow GLib License: Apache-2.0 -Requires: %{name}%{major_version}-libs = %{version}-%{release} +Requires: %{name}%{so_version}-libs = %{version}-%{release} -%description -n %{name}%{major_version}-glib-libs +%description -n %{name}%{so_version}-glib-libs This package contains the libraries for Apache Arrow GLib. -%files -n %{name}%{major_version}-glib-libs +%files -n %{name}%{so_version}-glib-libs %defattr(-,root,root,-) %doc README.md %license LICENSE.txt NOTICE.txt @@ -567,7 +569,7 @@ This package contains the libraries for Apache Arrow GLib. Summary: Libraries and header files for Apache Arrow GLib License: Apache-2.0 Requires: %{name}-acero-devel = %{version}-%{release} -Requires: %{name}%{major_version}-glib-libs = %{version}-%{release} +Requires: %{name}%{so_version}-glib-libs = %{version}-%{release} Requires: glib2-devel Requires: gobject-introspection-devel @@ -606,16 +608,16 @@ Documentation for Apache Arrow GLib. %{_docdir}/arrow-glib/ %endif -%package -n %{name}%{major_version}-dataset-glib-libs +%package -n %{name}%{so_version}-dataset-glib-libs Summary: Runtime libraries for Apache Arrow Dataset GLib License: Apache-2.0 -Requires: %{name}%{major_version}-dataset-libs = %{version}-%{release} -Requires: %{name}%{major_version}-glib-libs = %{version}-%{release} +Requires: %{name}%{so_version}-dataset-libs = %{version}-%{release} +Requires: %{name}%{so_version}-glib-libs = %{version}-%{release} -%description -n %{name}%{major_version}-dataset-glib-libs +%description -n %{name}%{so_version}-dataset-glib-libs This package contains the libraries for Apache Arrow Dataset GLib. -%files -n %{name}%{major_version}-dataset-glib-libs +%files -n %{name}%{so_version}-dataset-glib-libs %defattr(-,root,root,-) %doc README.md %license LICENSE.txt NOTICE.txt @@ -625,7 +627,7 @@ This package contains the libraries for Apache Arrow Dataset GLib. %package dataset-glib-devel Summary: Libraries and header files for Apache Arrow Dataset GLib License: Apache-2.0 -Requires: %{name}%{major_version}-dataset-glib-libs = %{version}-%{release} +Requires: %{name}%{so_version}-dataset-glib-libs = %{version}-%{release} Requires: %{name}-dataset-devel = %{version}-%{release} Requires: %{name}-glib-devel = %{version}-%{release} @@ -661,16 +663,16 @@ Documentation for Apache Arrow dataset GLib. %endif %if %{use_flight} -%package -n %{name}%{major_version}-flight-glib-libs +%package -n %{name}%{so_version}-flight-glib-libs Summary: Runtime libraries for Apache Arrow Flight GLib License: Apache-2.0 -Requires: %{name}%{major_version}-flight-libs = %{version}-%{release} -Requires: %{name}%{major_version}-glib-libs = %{version}-%{release} +Requires: %{name}%{so_version}-flight-libs = %{version}-%{release} +Requires: %{name}%{so_version}-glib-libs = %{version}-%{release} -%description -n %{name}%{major_version}-flight-glib-libs +%description -n %{name}%{so_version}-flight-glib-libs This package contains the libraries for Apache Arrow Flight GLib. -%files -n %{name}%{major_version}-flight-glib-libs +%files -n %{name}%{so_version}-flight-glib-libs %defattr(-,root,root,-) %doc README.md %license LICENSE.txt NOTICE.txt @@ -680,7 +682,7 @@ This package contains the libraries for Apache Arrow Flight GLib. %package flight-glib-devel Summary: Libraries and header files for Apache Arrow Flight GLib License: Apache-2.0 -Requires: %{name}%{major_version}-flight-glib-libs = %{version}-%{release} +Requires: %{name}%{so_version}-flight-glib-libs = %{version}-%{release} Requires: %{name}-flight-devel = %{version}-%{release} Requires: %{name}-glib-devel = %{version}-%{release} @@ -715,16 +717,16 @@ Documentation for Apache Arrow Flight GLib. %{_docdir}/arrow-flight-glib/ %endif -%package -n %{name}%{major_version}-flight-sql-glib-libs +%package -n %{name}%{so_version}-flight-sql-glib-libs Summary: Runtime libraries for Apache Arrow Flight SQL GLib License: Apache-2.0 -Requires: %{name}%{major_version}-flight-sql-libs = %{version}-%{release} -Requires: %{name}%{major_version}-flight-glib-libs = %{version}-%{release} +Requires: %{name}%{so_version}-flight-sql-libs = %{version}-%{release} +Requires: %{name}%{so_version}-flight-glib-libs = %{version}-%{release} -%description -n %{name}%{major_version}-flight-sql-glib-libs +%description -n %{name}%{so_version}-flight-sql-glib-libs This package contains the libraries for Apache Arrow Flight SQL GLib. -%files -n %{name}%{major_version}-flight-sql-glib-libs +%files -n %{name}%{so_version}-flight-sql-glib-libs %defattr(-,root,root,-) %doc README.md %license LICENSE.txt NOTICE.txt @@ -734,7 +736,7 @@ This package contains the libraries for Apache Arrow Flight SQL GLib. %package flight-sql-glib-devel Summary: Libraries and header files for Apache Arrow Flight SQL GLib License: Apache-2.0 -Requires: %{name}%{major_version}-flight-sql-glib-libs = %{version}-%{release} +Requires: %{name}%{so_version}-flight-sql-glib-libs = %{version}-%{release} Requires: %{name}-flight-sql-devel = %{version}-%{release} Requires: %{name}-flight-glib-devel = %{version}-%{release} @@ -771,16 +773,16 @@ Documentation for Apache Arrow Flight SQL GLib. %endif %if %{use_gandiva} -%package -n gandiva%{major_version}-glib-libs +%package -n gandiva%{so_version}-glib-libs Summary: Runtime libraries for Gandiva GLib License: Apache-2.0 -Requires: %{name}%{major_version}-glib-libs = %{version}-%{release} -Requires: gandiva%{major_version}-libs = %{version}-%{release} +Requires: %{name}%{so_version}-glib-libs = %{version}-%{release} +Requires: gandiva%{so_version}-libs = %{version}-%{release} -%description -n gandiva%{major_version}-glib-libs +%description -n gandiva%{so_version}-glib-libs This package contains the libraries for Gandiva GLib. -%files -n gandiva%{major_version}-glib-libs +%files -n gandiva%{so_version}-glib-libs %defattr(-,root,root,-) %doc README.md %license LICENSE.txt NOTICE.txt @@ -791,7 +793,7 @@ This package contains the libraries for Gandiva GLib. Summary: Libraries and header files for Gandiva GLib License: Apache-2.0 Requires: %{name}-glib-devel = %{version}-%{release} -Requires: gandiva%{major_version}-glib-libs = %{version}-%{release} +Requires: gandiva%{so_version}-glib-libs = %{version}-%{release} Requires: gandiva-devel = %{version}-%{release} %description -n gandiva-glib-devel @@ -826,16 +828,16 @@ Documentation for Gandiva GLib. %endif %endif -%package -n parquet%{major_version}-glib-libs +%package -n parquet%{so_version}-glib-libs Summary: Runtime libraries for Apache Parquet GLib License: Apache-2.0 -Requires: %{name}%{major_version}-glib-libs = %{version}-%{release} -Requires: parquet%{major_version}-libs = %{version}-%{release} +Requires: %{name}%{so_version}-glib-libs = %{version}-%{release} +Requires: parquet%{so_version}-libs = %{version}-%{release} -%description -n parquet%{major_version}-glib-libs +%description -n parquet%{so_version}-glib-libs This package contains the libraries for Apache Parquet GLib. -%files -n parquet%{major_version}-glib-libs +%files -n parquet%{so_version}-glib-libs %defattr(-,root,root,-) %doc README.md %license LICENSE.txt NOTICE.txt @@ -846,7 +848,7 @@ This package contains the libraries for Apache Parquet GLib. Summary: Libraries and header files for Apache Parquet GLib License: Apache-2.0 Requires: %{name}-glib-devel = %{version}-%{release} -Requires: parquet%{major_version}-glib-libs = %{version}-%{release} +Requires: parquet%{so_version}-glib-libs = %{version}-%{release} Requires: parquet-devel = %{version}-%{release} %description -n parquet-glib-devel From fe2d926ef385be58833f0e5e09d1860c63f800e6 Mon Sep 17 00:00:00 2001 From: Sarah Gilmore <74676073+sgilmore10@users.noreply.github.com> Date: Tue, 28 May 2024 09:37:54 -0400 Subject: [PATCH 18/93] GH-41803: [MATLAB] Add C Data Interface format import/export functionality for `arrow.tabular.RecordBatch` (#41817) ### Rationale for this change This pull requests adds two new APIs for importing and exporting `arrow.tabular.RecordBatch` instances using the C Data Interface format. **Example:** ```matlab >> T = table((1:3)', ["A"; "B"; "C"]); >> expected = arrow.recordBatch(T) expected = Arrow RecordBatch with 3 rows and 2 columns: Schema: Var1: Float64 | Var2: String First Row: 1 | "A" >> cArray = arrow.c.Array(); >> cSchema = arrow.c.Schema(); % Export the RecordBatch to C Data Interface Format >> expected.export(cArray.Address, cSchema.Address); % Import the RecordBatch from C Data Interface Format >> actual = arrow.tabular.RecordBatch.import(cArray, cSchema) actual = Arrow RecordBatch with 3 rows and 2 columns: Schema: Var1: Float64 | Var2: String First Row: 1 | "A" % The RecordBatch is the same after round-tripping to the C Data Interface format >> isequal(actual, expected) ans = logical 1 ``` ### What changes are included in this PR? 1. Added a new method `arrow.tabular.RecordBatch.export` for exporting `RecordBatch` objects to the C Data Interface format. 2. Added a new static method `arrow.tabular.RecordBatch.import` for importing `RecordBatch` objects from the C Data Interface format. 3. Added a new internal class `arrow.c.internal.RecordBatchImporter` for importing `RecordBatch` objects from the C Data Interface format. ### Are these changes tested? Yes. 1. Added a new test file `matlab/test/arrow/c/tRoundtripRecordBatch.m` which has basic round-trip tests for importing and exporting `RecordBatch` objects. ### Are there any user-facing changes? Yes. 1. Two new user-facing methods were added to `arrow.tabular.RecordBatch`. The first is `arrow.tabular.RecordBatch.export(cArrowArrayAddress, cArrowSchemaAddress)`. The second is `arrow.tabular.RecordBatch.import(cArray, cSchema)`. These APIs can be used to export/import `RecordBatch` objects using the C Data Interface format. ### Future Directions 1. Add integration tests for sharing data between MATLAB/mlarrow and Python/pyarrow running in the same process using the [MATLAB interface to Python](https://www.mathworks.com/help/matlab/call-python-libraries.html). 2. Add support for the Arrow [C stream interface format](https://arrow.apache.org/docs/format/CStreamInterface.html). ### Notes 1. Thanks to @ kevingurney for the help with this feature! * GitHub Issue: #41803 Authored-by: Sarah Gilmore Signed-off-by: Sarah Gilmore --- .../matlab/c/proxy/record_batch_importer.cc | 66 +++++++ .../matlab/c/proxy/record_batch_importer.h | 37 ++++ matlab/src/cpp/arrow/matlab/proxy/factory.cc | 104 +++++------ .../matlab/tabular/proxy/record_batch.cc | 19 +- .../arrow/matlab/tabular/proxy/record_batch.h | 1 + .../+arrow/+c/+internal/RecordBatchImporter.m | 52 ++++++ .../src/matlab/+arrow/+tabular/RecordBatch.m | 22 +++ matlab/test/arrow/c/tRoundTripRecordBatch.m | 170 ++++++++++++++++++ .../cmake/BuildMatlabArrowInterface.cmake | 3 +- 9 files changed, 420 insertions(+), 54 deletions(-) create mode 100644 matlab/src/cpp/arrow/matlab/c/proxy/record_batch_importer.cc create mode 100644 matlab/src/cpp/arrow/matlab/c/proxy/record_batch_importer.h create mode 100644 matlab/src/matlab/+arrow/+c/+internal/RecordBatchImporter.m create mode 100644 matlab/test/arrow/c/tRoundTripRecordBatch.m diff --git a/matlab/src/cpp/arrow/matlab/c/proxy/record_batch_importer.cc b/matlab/src/cpp/arrow/matlab/c/proxy/record_batch_importer.cc new file mode 100644 index 0000000000000..ed9ba14cfbe01 --- /dev/null +++ b/matlab/src/cpp/arrow/matlab/c/proxy/record_batch_importer.cc @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/c/bridge.h" + +#include "arrow/matlab/c/proxy/record_batch_importer.h" +#include "arrow/matlab/error/error.h" +#include "arrow/matlab/tabular/proxy/record_batch.h" + +#include "libmexclass/proxy/ProxyManager.h" + +namespace arrow::matlab::c::proxy { + +RecordBatchImporter::RecordBatchImporter() { + REGISTER_METHOD(RecordBatchImporter, import); +} + +libmexclass::proxy::MakeResult RecordBatchImporter::make( + const libmexclass::proxy::FunctionArguments& constructor_arguments) { + return std::make_shared(); +} + +void RecordBatchImporter::import(libmexclass::proxy::method::Context& context) { + namespace mda = ::matlab::data; + using namespace libmexclass::proxy; + using RecordBatchProxy = arrow::matlab::tabular::proxy::RecordBatch; + + mda::StructArray args = context.inputs[0]; + const mda::TypedArray arrow_array_address_mda = args[0]["ArrowArrayAddress"]; + const mda::TypedArray arrow_schema_address_mda = + args[0]["ArrowSchemaAddress"]; + + const auto arrow_array_address = uint64_t(arrow_array_address_mda[0]); + const auto arrow_schema_address = uint64_t(arrow_schema_address_mda[0]); + + auto arrow_array = reinterpret_cast(arrow_array_address); + auto arrow_schema = reinterpret_cast(arrow_schema_address); + + MATLAB_ASSIGN_OR_ERROR_WITH_CONTEXT(auto record_batch, + arrow::ImportRecordBatch(arrow_array, arrow_schema), + context, error::C_IMPORT_FAILED); + + auto record_batch_proxy = std::make_shared(std::move(record_batch)); + + mda::ArrayFactory factory; + const auto record_batch_proxy_id = ProxyManager::manageProxy(record_batch_proxy); + const auto record_batch_proxy_id_mda = factory.createScalar(record_batch_proxy_id); + + context.outputs[0] = record_batch_proxy_id_mda; +} + +} // namespace arrow::matlab::c::proxy diff --git a/matlab/src/cpp/arrow/matlab/c/proxy/record_batch_importer.h b/matlab/src/cpp/arrow/matlab/c/proxy/record_batch_importer.h new file mode 100644 index 0000000000000..0f697db0d25b0 --- /dev/null +++ b/matlab/src/cpp/arrow/matlab/c/proxy/record_batch_importer.h @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "libmexclass/proxy/Proxy.h" + +namespace arrow::matlab::c::proxy { + +class RecordBatchImporter : public libmexclass::proxy::Proxy { + public: + RecordBatchImporter(); + + ~RecordBatchImporter() = default; + + static libmexclass::proxy::MakeResult make( + const libmexclass::proxy::FunctionArguments& constructor_arguments); + + protected: + void import(libmexclass::proxy::method::Context& context); +}; + +} // namespace arrow::matlab::c::proxy diff --git a/matlab/src/cpp/arrow/matlab/proxy/factory.cc b/matlab/src/cpp/arrow/matlab/proxy/factory.cc index 9b95fcf128090..53a19da82e334 100644 --- a/matlab/src/cpp/arrow/matlab/proxy/factory.cc +++ b/matlab/src/cpp/arrow/matlab/proxy/factory.cc @@ -27,6 +27,7 @@ #include "arrow/matlab/buffer/proxy/buffer.h" #include "arrow/matlab/c/proxy/array.h" #include "arrow/matlab/c/proxy/array_importer.h" +#include "arrow/matlab/c/proxy/record_batch_importer.h" #include "arrow/matlab/c/proxy/schema.h" #include "arrow/matlab/error/error.h" #include "arrow/matlab/io/csv/proxy/table_reader.h" @@ -54,57 +55,58 @@ namespace arrow::matlab::proxy { libmexclass::proxy::MakeResult Factory::make_proxy( const ClassName& class_name, const FunctionArguments& constructor_arguments) { // clang-format off - REGISTER_PROXY(arrow.array.proxy.Float32Array , arrow::matlab::array::proxy::NumericArray); - REGISTER_PROXY(arrow.array.proxy.Float64Array , arrow::matlab::array::proxy::NumericArray); - REGISTER_PROXY(arrow.array.proxy.UInt8Array , arrow::matlab::array::proxy::NumericArray); - REGISTER_PROXY(arrow.array.proxy.UInt16Array , arrow::matlab::array::proxy::NumericArray); - REGISTER_PROXY(arrow.array.proxy.UInt32Array , arrow::matlab::array::proxy::NumericArray); - REGISTER_PROXY(arrow.array.proxy.UInt64Array , arrow::matlab::array::proxy::NumericArray); - REGISTER_PROXY(arrow.array.proxy.Int8Array , arrow::matlab::array::proxy::NumericArray); - REGISTER_PROXY(arrow.array.proxy.Int16Array , arrow::matlab::array::proxy::NumericArray); - REGISTER_PROXY(arrow.array.proxy.Int32Array , arrow::matlab::array::proxy::NumericArray); - REGISTER_PROXY(arrow.array.proxy.Int64Array , arrow::matlab::array::proxy::NumericArray); - REGISTER_PROXY(arrow.array.proxy.BooleanArray , arrow::matlab::array::proxy::BooleanArray); - REGISTER_PROXY(arrow.array.proxy.StringArray , arrow::matlab::array::proxy::StringArray); - REGISTER_PROXY(arrow.array.proxy.StructArray , arrow::matlab::array::proxy::StructArray); - REGISTER_PROXY(arrow.array.proxy.ListArray , arrow::matlab::array::proxy::ListArray); - REGISTER_PROXY(arrow.array.proxy.TimestampArray, arrow::matlab::array::proxy::NumericArray); - REGISTER_PROXY(arrow.array.proxy.Time32Array , arrow::matlab::array::proxy::NumericArray); - REGISTER_PROXY(arrow.array.proxy.Time64Array , arrow::matlab::array::proxy::NumericArray); - REGISTER_PROXY(arrow.array.proxy.Date32Array , arrow::matlab::array::proxy::NumericArray); - REGISTER_PROXY(arrow.array.proxy.Date64Array , arrow::matlab::array::proxy::NumericArray); - REGISTER_PROXY(arrow.array.proxy.ChunkedArray , arrow::matlab::array::proxy::ChunkedArray); - REGISTER_PROXY(arrow.buffer.proxy.Buffer , arrow::matlab::buffer::proxy::Buffer); - REGISTER_PROXY(arrow.tabular.proxy.RecordBatch , arrow::matlab::tabular::proxy::RecordBatch); - REGISTER_PROXY(arrow.tabular.proxy.Table , arrow::matlab::tabular::proxy::Table); - REGISTER_PROXY(arrow.tabular.proxy.Schema , arrow::matlab::tabular::proxy::Schema); - REGISTER_PROXY(arrow.type.proxy.Field , arrow::matlab::type::proxy::Field); - REGISTER_PROXY(arrow.type.proxy.Float32Type , arrow::matlab::type::proxy::PrimitiveCType); - REGISTER_PROXY(arrow.type.proxy.Float64Type , arrow::matlab::type::proxy::PrimitiveCType); - REGISTER_PROXY(arrow.type.proxy.UInt8Type , arrow::matlab::type::proxy::PrimitiveCType); - REGISTER_PROXY(arrow.type.proxy.UInt16Type , arrow::matlab::type::proxy::PrimitiveCType); - REGISTER_PROXY(arrow.type.proxy.UInt32Type , arrow::matlab::type::proxy::PrimitiveCType); - REGISTER_PROXY(arrow.type.proxy.UInt64Type , arrow::matlab::type::proxy::PrimitiveCType); - REGISTER_PROXY(arrow.type.proxy.Int8Type , arrow::matlab::type::proxy::PrimitiveCType); - REGISTER_PROXY(arrow.type.proxy.Int16Type , arrow::matlab::type::proxy::PrimitiveCType); - REGISTER_PROXY(arrow.type.proxy.Int32Type , arrow::matlab::type::proxy::PrimitiveCType); - REGISTER_PROXY(arrow.type.proxy.Int64Type , arrow::matlab::type::proxy::PrimitiveCType); - REGISTER_PROXY(arrow.type.proxy.BooleanType , arrow::matlab::type::proxy::PrimitiveCType); - REGISTER_PROXY(arrow.type.proxy.StringType , arrow::matlab::type::proxy::StringType); - REGISTER_PROXY(arrow.type.proxy.TimestampType , arrow::matlab::type::proxy::TimestampType); - REGISTER_PROXY(arrow.type.proxy.Time32Type , arrow::matlab::type::proxy::Time32Type); - REGISTER_PROXY(arrow.type.proxy.Time64Type , arrow::matlab::type::proxy::Time64Type); - REGISTER_PROXY(arrow.type.proxy.Date32Type , arrow::matlab::type::proxy::Date32Type); - REGISTER_PROXY(arrow.type.proxy.Date64Type , arrow::matlab::type::proxy::Date64Type); - REGISTER_PROXY(arrow.type.proxy.StructType , arrow::matlab::type::proxy::StructType); - REGISTER_PROXY(arrow.type.proxy.ListType , arrow::matlab::type::proxy::ListType); - REGISTER_PROXY(arrow.io.feather.proxy.Writer , arrow::matlab::io::feather::proxy::Writer); - REGISTER_PROXY(arrow.io.feather.proxy.Reader , arrow::matlab::io::feather::proxy::Reader); - REGISTER_PROXY(arrow.io.csv.proxy.TableWriter , arrow::matlab::io::csv::proxy::TableWriter); - REGISTER_PROXY(arrow.io.csv.proxy.TableReader , arrow::matlab::io::csv::proxy::TableReader); - REGISTER_PROXY(arrow.c.proxy.Array , arrow::matlab::c::proxy::Array); - REGISTER_PROXY(arrow.c.proxy.ArrayImporter , arrow::matlab::c::proxy::ArrayImporter); - REGISTER_PROXY(arrow.c.proxy.Schema , arrow::matlab::c::proxy::Schema); + REGISTER_PROXY(arrow.array.proxy.Float32Array , arrow::matlab::array::proxy::NumericArray); + REGISTER_PROXY(arrow.array.proxy.Float64Array , arrow::matlab::array::proxy::NumericArray); + REGISTER_PROXY(arrow.array.proxy.UInt8Array , arrow::matlab::array::proxy::NumericArray); + REGISTER_PROXY(arrow.array.proxy.UInt16Array , arrow::matlab::array::proxy::NumericArray); + REGISTER_PROXY(arrow.array.proxy.UInt32Array , arrow::matlab::array::proxy::NumericArray); + REGISTER_PROXY(arrow.array.proxy.UInt64Array , arrow::matlab::array::proxy::NumericArray); + REGISTER_PROXY(arrow.array.proxy.Int8Array , arrow::matlab::array::proxy::NumericArray); + REGISTER_PROXY(arrow.array.proxy.Int16Array , arrow::matlab::array::proxy::NumericArray); + REGISTER_PROXY(arrow.array.proxy.Int32Array , arrow::matlab::array::proxy::NumericArray); + REGISTER_PROXY(arrow.array.proxy.Int64Array , arrow::matlab::array::proxy::NumericArray); + REGISTER_PROXY(arrow.array.proxy.BooleanArray , arrow::matlab::array::proxy::BooleanArray); + REGISTER_PROXY(arrow.array.proxy.StringArray , arrow::matlab::array::proxy::StringArray); + REGISTER_PROXY(arrow.array.proxy.StructArray , arrow::matlab::array::proxy::StructArray); + REGISTER_PROXY(arrow.array.proxy.ListArray , arrow::matlab::array::proxy::ListArray); + REGISTER_PROXY(arrow.array.proxy.TimestampArray , arrow::matlab::array::proxy::NumericArray); + REGISTER_PROXY(arrow.array.proxy.Time32Array , arrow::matlab::array::proxy::NumericArray); + REGISTER_PROXY(arrow.array.proxy.Time64Array , arrow::matlab::array::proxy::NumericArray); + REGISTER_PROXY(arrow.array.proxy.Date32Array , arrow::matlab::array::proxy::NumericArray); + REGISTER_PROXY(arrow.array.proxy.Date64Array , arrow::matlab::array::proxy::NumericArray); + REGISTER_PROXY(arrow.array.proxy.ChunkedArray , arrow::matlab::array::proxy::ChunkedArray); + REGISTER_PROXY(arrow.buffer.proxy.Buffer , arrow::matlab::buffer::proxy::Buffer); + REGISTER_PROXY(arrow.tabular.proxy.RecordBatch , arrow::matlab::tabular::proxy::RecordBatch); + REGISTER_PROXY(arrow.tabular.proxy.Table , arrow::matlab::tabular::proxy::Table); + REGISTER_PROXY(arrow.tabular.proxy.Schema , arrow::matlab::tabular::proxy::Schema); + REGISTER_PROXY(arrow.type.proxy.Field , arrow::matlab::type::proxy::Field); + REGISTER_PROXY(arrow.type.proxy.Float32Type , arrow::matlab::type::proxy::PrimitiveCType); + REGISTER_PROXY(arrow.type.proxy.Float64Type , arrow::matlab::type::proxy::PrimitiveCType); + REGISTER_PROXY(arrow.type.proxy.UInt8Type , arrow::matlab::type::proxy::PrimitiveCType); + REGISTER_PROXY(arrow.type.proxy.UInt16Type , arrow::matlab::type::proxy::PrimitiveCType); + REGISTER_PROXY(arrow.type.proxy.UInt32Type , arrow::matlab::type::proxy::PrimitiveCType); + REGISTER_PROXY(arrow.type.proxy.UInt64Type , arrow::matlab::type::proxy::PrimitiveCType); + REGISTER_PROXY(arrow.type.proxy.Int8Type , arrow::matlab::type::proxy::PrimitiveCType); + REGISTER_PROXY(arrow.type.proxy.Int16Type , arrow::matlab::type::proxy::PrimitiveCType); + REGISTER_PROXY(arrow.type.proxy.Int32Type , arrow::matlab::type::proxy::PrimitiveCType); + REGISTER_PROXY(arrow.type.proxy.Int64Type , arrow::matlab::type::proxy::PrimitiveCType); + REGISTER_PROXY(arrow.type.proxy.BooleanType , arrow::matlab::type::proxy::PrimitiveCType); + REGISTER_PROXY(arrow.type.proxy.StringType , arrow::matlab::type::proxy::StringType); + REGISTER_PROXY(arrow.type.proxy.TimestampType , arrow::matlab::type::proxy::TimestampType); + REGISTER_PROXY(arrow.type.proxy.Time32Type , arrow::matlab::type::proxy::Time32Type); + REGISTER_PROXY(arrow.type.proxy.Time64Type , arrow::matlab::type::proxy::Time64Type); + REGISTER_PROXY(arrow.type.proxy.Date32Type , arrow::matlab::type::proxy::Date32Type); + REGISTER_PROXY(arrow.type.proxy.Date64Type , arrow::matlab::type::proxy::Date64Type); + REGISTER_PROXY(arrow.type.proxy.StructType , arrow::matlab::type::proxy::StructType); + REGISTER_PROXY(arrow.type.proxy.ListType , arrow::matlab::type::proxy::ListType); + REGISTER_PROXY(arrow.io.feather.proxy.Writer , arrow::matlab::io::feather::proxy::Writer); + REGISTER_PROXY(arrow.io.feather.proxy.Reader , arrow::matlab::io::feather::proxy::Reader); + REGISTER_PROXY(arrow.io.csv.proxy.TableWriter , arrow::matlab::io::csv::proxy::TableWriter); + REGISTER_PROXY(arrow.io.csv.proxy.TableReader , arrow::matlab::io::csv::proxy::TableReader); + REGISTER_PROXY(arrow.c.proxy.Array , arrow::matlab::c::proxy::Array); + REGISTER_PROXY(arrow.c.proxy.ArrayImporter , arrow::matlab::c::proxy::ArrayImporter); + REGISTER_PROXY(arrow.c.proxy.Schema , arrow::matlab::c::proxy::Schema); + REGISTER_PROXY(arrow.c.proxy.RecordBatchImporter , arrow::matlab::c::proxy::RecordBatchImporter); // clang-format on return libmexclass::error::Error{error::UNKNOWN_PROXY_ERROR_ID, diff --git a/matlab/src/cpp/arrow/matlab/tabular/proxy/record_batch.cc b/matlab/src/cpp/arrow/matlab/tabular/proxy/record_batch.cc index 298ac4b595139..f3cee25a3a8ee 100644 --- a/matlab/src/cpp/arrow/matlab/tabular/proxy/record_batch.cc +++ b/matlab/src/cpp/arrow/matlab/tabular/proxy/record_batch.cc @@ -15,8 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "libmexclass/proxy/ProxyManager.h" - +#include "arrow/c/bridge.h" #include "arrow/matlab/array/proxy/array.h" #include "arrow/matlab/array/proxy/wrap.h" @@ -66,6 +65,7 @@ RecordBatch::RecordBatch(std::shared_ptr record_batch) REGISTER_METHOD(RecordBatch, getColumnByName); REGISTER_METHOD(RecordBatch, getSchema); REGISTER_METHOD(RecordBatch, getRowAsString); + REGISTER_METHOD(RecordBatch, exportToC); } std::shared_ptr RecordBatch::unwrap() { return record_batch; } @@ -259,4 +259,19 @@ void RecordBatch::getRowAsString(libmexclass::proxy::method::Context& context) { context.outputs[0] = factory.createScalar(row_str_utf16); } +void RecordBatch::exportToC(libmexclass::proxy::method::Context& context) { + namespace mda = ::matlab::data; + mda::StructArray opts = context.inputs[0]; + const mda::TypedArray array_address_mda = opts[0]["ArrowArrayAddress"]; + const mda::TypedArray schema_address_mda = opts[0]["ArrowSchemaAddress"]; + + auto arrow_array = reinterpret_cast(uint64_t(array_address_mda[0])); + auto arrow_schema = + reinterpret_cast(uint64_t(schema_address_mda[0])); + + MATLAB_ERROR_IF_NOT_OK_WITH_CONTEXT( + arrow::ExportRecordBatch(*record_batch, arrow_array, arrow_schema), context, + error::C_EXPORT_FAILED); +} + } // namespace arrow::matlab::tabular::proxy diff --git a/matlab/src/cpp/arrow/matlab/tabular/proxy/record_batch.h b/matlab/src/cpp/arrow/matlab/tabular/proxy/record_batch.h index c8285c9b095d5..4a1675a8a438a 100644 --- a/matlab/src/cpp/arrow/matlab/tabular/proxy/record_batch.h +++ b/matlab/src/cpp/arrow/matlab/tabular/proxy/record_batch.h @@ -43,6 +43,7 @@ class RecordBatch : public libmexclass::proxy::Proxy { void getColumnByName(libmexclass::proxy::method::Context& context); void getSchema(libmexclass::proxy::method::Context& context); void getRowAsString(libmexclass::proxy::method::Context& context); + void exportToC(libmexclass::proxy::method::Context& context); std::shared_ptr record_batch; }; diff --git a/matlab/src/matlab/+arrow/+c/+internal/RecordBatchImporter.m b/matlab/src/matlab/+arrow/+c/+internal/RecordBatchImporter.m new file mode 100644 index 0000000000000..120763bb46e7b --- /dev/null +++ b/matlab/src/matlab/+arrow/+c/+internal/RecordBatchImporter.m @@ -0,0 +1,52 @@ +%RECORDBATCHIMPORTER Imports Arrow RecordBatch using the C Data Interface +% Format. + +% Licensed to the Apache Software Foundation (ASF) under one or more +% contributor license agreements. See the NOTICE file distributed with +% this work for additional information regarding copyright ownership. +% The ASF licenses this file to you under the Apache License, Version +% 2.0 (the "License"); you may not use this file except in compliance +% with the License. You may obtain a copy of the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, +% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +% implied. See the License for the specific language governing +% permissions and limitations under the License. + +classdef RecordBatchImporter + + properties (Hidden, SetAccess=private, GetAccess=public) + Proxy + end + + methods + + function obj = RecordBatchImporter() + proxyName = "arrow.c.proxy.RecordBatchImporter"; + proxy = arrow.internal.proxy.create(proxyName, struct()); + obj.Proxy = proxy; + end + + function recordBatch = import(obj, cArray, cSchema) + arguments + obj(1, 1) arrow.c.internal.RecordBatchImporter + cArray(1, 1) arrow.c.Array + cSchema(1, 1) arrow.c.Schema + end + args = struct(... + ArrowArrayAddress=cArray.Address,... + ArrowSchemaAddress=cSchema.Address... + ); + proxyID = obj.Proxy.import(args); + proxyName = "arrow.tabular.proxy.RecordBatch"; + proxy = libmexclass.proxy.Proxy(Name=proxyName, ID=proxyID); + recordBatch = arrow.tabular.RecordBatch(proxy); + end + + end + +end + diff --git a/matlab/src/matlab/+arrow/+tabular/RecordBatch.m b/matlab/src/matlab/+arrow/+tabular/RecordBatch.m index 0225f3d771181..da5c1fc1c3764 100644 --- a/matlab/src/matlab/+arrow/+tabular/RecordBatch.m +++ b/matlab/src/matlab/+arrow/+tabular/RecordBatch.m @@ -102,6 +102,19 @@ function tf = isequal(obj, varargin) tf = arrow.tabular.internal.isequal(obj, varargin{:}); end + + function export(obj, cArrowArrayAddress, cArrowSchemaAddress) + arguments + obj(1, 1) arrow.tabular.RecordBatch + cArrowArrayAddress(1, 1) uint64 + cArrowSchemaAddress(1, 1) uint64 + end + args = struct(... + ArrowArrayAddress=cArrowArrayAddress,... + ArrowSchemaAddress=cArrowSchemaAddress... + ); + obj.Proxy.exportToC(args); + end end methods (Access = private) @@ -141,5 +154,14 @@ function displayScalarObject(obj) proxy = arrow.internal.proxy.create(proxyName, args); recordBatch = arrow.tabular.RecordBatch(proxy); end + + function recordBatch = import(cArray, cSchema) + arguments + cArray(1, 1) arrow.c.Array + cSchema(1, 1) arrow.c.Schema + end + importer = arrow.c.internal.RecordBatchImporter(); + recordBatch = importer.import(cArray, cSchema); + end end end diff --git a/matlab/test/arrow/c/tRoundTripRecordBatch.m b/matlab/test/arrow/c/tRoundTripRecordBatch.m new file mode 100644 index 0000000000000..5d95aecbe1603 --- /dev/null +++ b/matlab/test/arrow/c/tRoundTripRecordBatch.m @@ -0,0 +1,170 @@ +%TROUNDTRIPRECORDBATCH Tests for roundtripping RecordBatches using +% the C Data Interface format. + +% Licensed to the Apache Software Foundation (ASF) under one or more +% contributor license agreements. See the NOTICE file distributed with +% this work for additional information regarding copyright ownership. +% The ASF licenses this file to you under the Apache License, Version +% 2.0 (the "License"); you may not use this file except in compliance +% with the License. You may obtain a copy of the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, +% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +% implied. See the License for the specific language governing +% permissions and limitations under the License. +classdef tRoundTripRecordBatch < matlab.unittest.TestCase + + methods (Test) + function ZeroColumnRecordBatch(testCase) + expected = arrow.recordBatch(table()); + + cArray = arrow.c.Array(); + cSchema = arrow.c.Schema(); + expected.export(cArray.Address, cSchema.Address); + actual = arrow.tabular.RecordBatch.import(cArray, cSchema); + + testCase.verifyEqual(actual, expected); + end + + function ZeroRowRecordBatch(testCase) + doubleArray = arrow.array([]); + stringArray = arrow.array(string.empty(0, 0)); + expected = arrow.tabular.RecordBatch.fromArrays(doubleArray, stringArray); + + cArray = arrow.c.Array(); + cSchema = arrow.c.Schema(); + expected.export(cArray.Address, cSchema.Address); + actual = arrow.tabular.RecordBatch.import(cArray, cSchema); + + testCase.verifyEqual(actual, expected); + end + + function OneRowRecordBatch(testCase) + varNames = ["Col1" "Col2" "Col3"]; + t = table(1, "A", false, VariableNames=varNames); + expected = arrow.recordBatch(t); + + cArray = arrow.c.Array(); + cSchema = arrow.c.Schema(); + expected.export(cArray.Address, cSchema.Address); + actual = arrow.tabular.RecordBatch.import(cArray, cSchema); + + testCase.verifyEqual(actual, expected); + end + + function MultiRowRecordBatch(testCase) + varNames = ["Col1" "Col2" "Col3"]; + t = table((1:3)', ["A"; "B"; "C"], [false; true; false],... + VariableNames=varNames); + expected = arrow.recordBatch(t); + + cArray = arrow.c.Array(); + cSchema = arrow.c.Schema(); + expected.export(cArray.Address, cSchema.Address); + actual = arrow.tabular.RecordBatch.import(cArray, cSchema); + + testCase.verifyEqual(actual, expected); + end + + function ExportErrorWrongInputTypes(testCase) + rb = arrow.recordBatch(table([1; 2; 3])); + fcn = @() rb.export("cArray.Address", "cSchema.Address"); + testCase.verifyError(fcn, "MATLAB:validation:UnableToConvert"); + end + + function ExportTooFewInputs(testCase) + rb = arrow.recordBatch(table([1; 2; 3])); + fcn = @() rb.export(); + testCase.verifyError(fcn, "MATLAB:minrhs"); + end + + function ExportTooManyInputs(testCase) + rb = arrow.recordBatch(table([1; 2; 3])); + fcn = @() rb.export("A", "B", "C"); + testCase.verifyError(fcn, "MATLAB:TooManyInputs"); + end + + function ImportErrorWrongInputTypes(testCase) + cArray = "arrow.c.Array"; + cSchema = "arrow.c.Schema"; + fcn = @() arrow.tabular.RecordBatch.import(cArray, cSchema); + testCase.verifyError(fcn, "MATLAB:validation:UnableToConvert"); + end + + function ImportTooFewInputs(testCase) + fcn = @() arrow.tabular.RecordBatch.import(); + testCase.verifyError(fcn, "MATLAB:minrhs"); + end + + function ImportTooManyInputs(testCase) + fcn = @() arrow.tabular.RecordBatch.import("A", "B", "C"); + testCase.verifyError(fcn, "MATLAB:TooManyInputs"); + end + + function ImportErrorImportFailed(testCase) + cArray = arrow.c.Array(); + cSchema = arrow.c.Schema(); + % An arrow:c:import:ImportFailed error should be thrown + % if the supplied arrow.c.Array and arrow.c.Schema were + % never populated previously from an exported Array. + fcn = @() arrow.tabular.RecordBatch.import(cArray, cSchema); + testCase.verifyError(fcn, "arrow:c:import:ImportFailed"); + end + + function ImportErrorInvalidSchema(testCase) + cArray = arrow.c.Array(); + cSchema = arrow.c.Schema(); + % An arrow:c:import:ImportFailed error should be thrown + % if the supplied arrow.c.Schema was not populated from a + % struct-like type (i.e. StructArray or RecordBatch). + a = arrow.array(1:3); + a.export(cArray.Address, cSchema.Address); + fcn = @() arrow.tabular.RecordBatch.import(cArray, cSchema); + testCase.verifyError(fcn, "arrow:c:import:ImportFailed"); + end + + function ImportFromStructArray(testCase) + % Verify a StructArray exported via the C Data Interface format + % can be imported as a RecordBatch. + field1 = arrow.array(1:3); + + field2 = arrow.array(["A" "B" "C"]); + structArray = arrow.array.StructArray.fromArrays(field1, field2, ... + FieldNames=["Number" "Text"]); + + cArray = arrow.c.Array(); + cSchema = arrow.c.Schema(); + structArray.export(cArray.Address, cSchema.Address) + rb = arrow.tabular.RecordBatch.import(cArray, cSchema); + + expected = arrow.tabular.RecordBatch.fromArrays(field1, field2, ... + ColumnNames=["Number" "Text"]); + + testCase.verifyEqual(rb, expected); + end + + function ExportToStructArray(testCase) + % Verify a RecordBatch exported via the C Data Interface + % format can be imported as a StructArray. + column1 = arrow.array(1:3); + column2 = arrow.array(["A" "B" "C"]); + rb = arrow.tabular.RecordBatch.fromArrays(column1, column2, ... + ColumnNames=["Number" "Text"]); + + cArray = arrow.c.Array(); + cSchema = arrow.c.Schema(); + rb.export(cArray.Address, cSchema.Address) + structArray = arrow.array.Array.import(cArray, cSchema); + + expected = arrow.array.StructArray.fromArrays(column1, column2, ... + FieldNames=["Number" "Text"]); + + testCase.verifyEqual(structArray, expected); + end + + end + +end \ No newline at end of file diff --git a/matlab/tools/cmake/BuildMatlabArrowInterface.cmake b/matlab/tools/cmake/BuildMatlabArrowInterface.cmake index 92e9f59145acc..0a747e648cd84 100644 --- a/matlab/tools/cmake/BuildMatlabArrowInterface.cmake +++ b/matlab/tools/cmake/BuildMatlabArrowInterface.cmake @@ -78,7 +78,8 @@ set(MATLAB_ARROW_LIBMEXCLASS_CLIENT_PROXY_SOURCES "${CMAKE_SOURCE_DIR}/src/cpp/a "${CMAKE_SOURCE_DIR}/src/cpp/arrow/matlab/buffer/proxy/buffer.cc" "${CMAKE_SOURCE_DIR}/src/cpp/arrow/matlab/c/proxy/array.cc" "${CMAKE_SOURCE_DIR}/src/cpp/arrow/matlab/c/proxy/array_importer.cc" - "${CMAKE_SOURCE_DIR}/src/cpp/arrow/matlab/c/proxy/schema.cc") + "${CMAKE_SOURCE_DIR}/src/cpp/arrow/matlab/c/proxy/schema.cc" + "${CMAKE_SOURCE_DIR}/src/cpp/arrow/matlab/c/proxy/record_batch_importer.cc") set(MATLAB_ARROW_LIBMEXCLASS_CLIENT_PROXY_FACTORY_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/src/cpp/arrow/matlab/proxy") From e6e00e7f5fb03cf4397496e2f7a0e9dc4da5126d Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 29 May 2024 00:02:19 +0900 Subject: [PATCH 19/93] GH-41771: [C++] Iterator releases its resource immediately when it reads all values (#41824) ### Rationale for this change `Iterator` keeps its resource (`ptr_`) until it's deleted but we can release its resource immediately when it reads all values. If `Iterator` keeps its resource until it's deleted, it may block closing a file. See GH-41771 for this case. ### What changes are included in this PR? Releases `ptr_` when `Next()` returns the end. ### Are these changes tested? Yes. ### Are there any user-facing changes? Yes. * GitHub Issue: #41771 Authored-by: Sutou Kouhei Signed-off-by: Benjamin Kietzman --- cpp/src/arrow/util/iterator.h | 15 ++++++++-- cpp/src/arrow/util/iterator_test.cc | 43 +++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/cpp/src/arrow/util/iterator.h b/cpp/src/arrow/util/iterator.h index 4da8394a0299c..5025799b9a372 100644 --- a/cpp/src/arrow/util/iterator.h +++ b/cpp/src/arrow/util/iterator.h @@ -105,9 +105,18 @@ class Iterator : public util::EqualityComparable> { Iterator() : ptr_(NULLPTR, [](void*) {}) {} /// \brief Return the next element of the sequence, IterationTraits::End() when the - /// iteration is completed. Calling this on a default constructed Iterator - /// will result in undefined behavior. - Result Next() { return next_(ptr_.get()); } + /// iteration is completed. + Result Next() { + if (ptr_) { + auto next_result = next_(ptr_.get()); + if (next_result.ok() && IsIterationEnd(next_result.ValueUnsafe())) { + ptr_.reset(NULLPTR); + } + return next_result; + } else { + return IterationTraits::End(); + } + } /// Pass each element of the sequence to a visitor. Will return any error status /// returned by the visitor, terminating iteration. diff --git a/cpp/src/arrow/util/iterator_test.cc b/cpp/src/arrow/util/iterator_test.cc index ba21ddcced209..a247ba13aef73 100644 --- a/cpp/src/arrow/util/iterator_test.cc +++ b/cpp/src/arrow/util/iterator_test.cc @@ -146,6 +146,49 @@ void AssertIteratorNext(T expected, Iterator& it) { ASSERT_EQ(expected, actual); } +template +class DeleteDetectableIterator { + public: + explicit DeleteDetectableIterator(std::vector values, bool* deleted) + : values_(std::move(values)), i_(0), deleted_(deleted) {} + + DeleteDetectableIterator(DeleteDetectableIterator&& source) + : values_(std::move(source.values_)), i_(source.i_), deleted_(source.deleted_) { + source.deleted_ = nullptr; + } + + ~DeleteDetectableIterator() { + if (deleted_) { + *deleted_ = true; + } + } + + Result Next() { + if (i_ == values_.size()) { + return IterationTraits::End(); + } + return std::move(values_[i_++]); + } + + private: + std::vector values_; + size_t i_; + bool* deleted_; +}; + +// Generic iterator tests + +TEST(TestIterator, DeleteOnEnd) { + bool deleted = false; + Iterator it(DeleteDetectableIterator({1}, &deleted)); + ASSERT_FALSE(deleted); + AssertIteratorNext({1}, it); + ASSERT_FALSE(deleted); + ASSERT_OK_AND_ASSIGN(auto value, it.Next()); + ASSERT_TRUE(IsIterationEnd(value)); + ASSERT_TRUE(deleted); +} + // -------------------------------------------------------------------- // Synchronous iterator tests From 8f3bf67cca32902e241b1857502247918861a3f8 Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Tue, 28 May 2024 17:26:09 -0500 Subject: [PATCH 20/93] GH-41841: [R][CI] Remove more defunct rhub containers (#41828) Testing CI to see if we can replicate the incoming NOTEs: ``` Found the following (possibly) invalid file URIs: URI: articles/read_write.html From: README.md URI: articles/data_wrangling.html From: README.md URI: reference/acero.html From: README.md URI: articles/install.html From: README.md URI: articles/install_nightly.html From: README.md ``` I wasn't able to replicate them in CI (even with `_R_CHECK_CRAN_INCOMING_REMOTE_` set to true, and installing pandoc so that the docs could be munged.) But in the process realized we were running old rhub images that aren't updated anymore (thanks, @ thisisnic). Also did a bit of cleanup of `--run-donttest` which is now no longer needed (was removed in favor of the env var in 4.0) * GitHub Issue: #41841 Authored-by: Jonathan Keane Signed-off-by: Jonathan Keane --- .github/workflows/r.yml | 5 ++- ci/scripts/r_install_system_dependencies.sh | 43 +++++++++++---------- ci/scripts/r_test.sh | 9 ++--- dev/tasks/r/github.linux.cran.yml | 9 ++--- r/Makefile | 4 +- 5 files changed, 35 insertions(+), 35 deletions(-) diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index aba77347659cd..6bd940f806775 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -370,11 +370,12 @@ jobs: MAKEFLAGS = paste0("-j", parallel::detectCores()), ARROW_R_DEV = TRUE, "_R_CHECK_FORCE_SUGGESTS_" = FALSE, - "_R_CHECK_STOP_ON_INVALID_NUMERIC_VERSION_INPUTS_" = TRUE + "_R_CHECK_STOP_ON_INVALID_NUMERIC_VERSION_INPUTS_" = TRUE, + "_R_CHECK_DONTTEST_EXAMPLES_" = TRUE ) rcmdcheck::rcmdcheck(".", build_args = '--no-build-vignettes', - args = c('--no-manual', '--as-cran', '--ignore-vignettes', '--run-donttest'), + args = c('--no-manual', '--as-cran', '--ignore-vignettes'), error_on = 'warning', check_dir = 'check', timeout = 3600 diff --git a/ci/scripts/r_install_system_dependencies.sh b/ci/scripts/r_install_system_dependencies.sh index be0d75ef235e6..7ddc2604f661a 100755 --- a/ci/scripts/r_install_system_dependencies.sh +++ b/ci/scripts/r_install_system_dependencies.sh @@ -21,29 +21,30 @@ set -ex : ${ARROW_SOURCE_HOME:=/arrow} -if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_GCS" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then - # Figure out what package manager we have - if [ "`which dnf`" ]; then - PACKAGE_MANAGER=dnf - elif [ "`which yum`" ]; then - PACKAGE_MANAGER=yum - elif [ "`which zypper`" ]; then - PACKAGE_MANAGER=zypper - else - PACKAGE_MANAGER=apt-get - apt-get update - fi +# Figure out what package manager we have +if [ "`which dnf`" ]; then + PACKAGE_MANAGER=dnf +elif [ "`which yum`" ]; then + PACKAGE_MANAGER=yum +elif [ "`which zypper`" ]; then + PACKAGE_MANAGER=zypper +else + PACKAGE_MANAGER=apt-get + apt-get update +fi - # Install curl and OpenSSL for S3/GCS support - case "$PACKAGE_MANAGER" in - apt-get) - apt-get install -y libcurl4-openssl-dev libssl-dev - ;; - *) - $PACKAGE_MANAGER install -y libcurl-devel openssl-devel - ;; - esac +# Install curl and OpenSSL (technically, only needed for S3/GCS support, but +# installing the R curl package fails without it) +case "$PACKAGE_MANAGER" in + apt-get) + apt-get install -y libcurl4-openssl-dev libssl-dev + ;; + *) + $PACKAGE_MANAGER install -y libcurl-devel openssl-devel + ;; +esac +if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_GCS" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then # The Dockerfile should have put this file here if [ "$ARROW_S3" == "ON" ] && [ -f "${ARROW_SOURCE_HOME}/ci/scripts/install_minio.sh" ] && [ "`which wget`" ]; then "${ARROW_SOURCE_HOME}/ci/scripts/install_minio.sh" latest /usr/local diff --git a/ci/scripts/r_test.sh b/ci/scripts/r_test.sh index e13da45e2d296..fe9d18edb8cbb 100755 --- a/ci/scripts/r_test.sh +++ b/ci/scripts/r_test.sh @@ -110,16 +110,15 @@ SCRIPT="as_cran <- !identical(tolower(Sys.getenv('NOT_CRAN')), 'true') on.exit(tools::pskill(pid_flight), add = TRUE) } - run_donttest <- identical(tolower(Sys.getenv('_R_CHECK_DONTTEST_EXAMPLES_', 'true')), 'true') - if (run_donttest) { - args <- c(args, '--run-donttest') - } - install_args <- Sys.getenv('INSTALL_ARGS') if (nzchar(install_args)) { args <- c(args, paste0('--install-args=\"', install_args, '\"')) } + message('Running rcmdcheck with:\n') + print(build_args) + print(args) + rcmdcheck::rcmdcheck(build_args = build_args, args = args, error_on = 'warning', check_dir = 'check', timeout = 3600)" echo "$SCRIPT" | ${R_BIN} --no-save diff --git a/dev/tasks/r/github.linux.cran.yml b/dev/tasks/r/github.linux.cran.yml index 0aeb7cfa2b434..34cb4b9446a0b 100644 --- a/dev/tasks/r/github.linux.cran.yml +++ b/dev/tasks/r/github.linux.cran.yml @@ -28,11 +28,10 @@ jobs: matrix: # See https://hub.docker.com/r/rhub r_image: - - debian-gcc-devel - - debian-gcc-patched - - debian-gcc-release - - fedora-gcc-devel - - fedora-clang-devel + - ubuntu-gcc12 # ~ r-devel-linux-x86_64-debian-gcc + - ubuntu-clang # ~ r-devel-linux-x86_64-debian-clang + - ubuntu-next # ~ r-patched-linux-x86_64 + - ubuntu-release # ~ r-release-linux-x86_64 env: R_ORG: "rhub" R_IMAGE: {{ MATRIX }} diff --git a/r/Makefile b/r/Makefile index c3267e8cfe45b..785e9e1214d4f 100644 --- a/r/Makefile +++ b/r/Makefile @@ -52,11 +52,11 @@ build: doc sync-cpp R CMD build ${args} . check: build - -export _R_CHECK_CRAN_INCOMING_REMOTE_=FALSE && export ARROW_R_DEV=$(ARROW_R_DEV) && export _R_CHECK_TESTS_NLINES_=0 && R CMD check --as-cran --run-donttest arrow_$(VERSION).tar.gz + -export _R_CHECK_CRAN_INCOMING_REMOTE_=FALSE && export ARROW_R_DEV=$(ARROW_R_DEV) && export _R_CHECK_TESTS_NLINES_=0 && R CMD check --as-cran arrow_$(VERSION).tar.gz rm -rf arrow.Rcheck/ release: build - -export _R_CHECK_TESTS_NLINES_=0 && R CMD check --as-cran --run-donttest arrow_$(VERSION).tar.gz + -export _R_CHECK_TESTS_NLINES_=0 && R CMD check --as-cran arrow_$(VERSION).tar.gz rm -rf arrow.Rcheck/ clean: From 235608beb68332d8e00f70afced0d2a7a52b2d98 Mon Sep 17 00:00:00 2001 From: Matt Topol Date: Tue, 28 May 2024 19:23:47 -0400 Subject: [PATCH 21/93] MINOR: [C++] Slight improvement for ArrayData device_type (#41814) Responding to feedback on #40807: This condition implies that, conversely, in non-debug mode we could immediately return when we encounter a buffer? Instead of continue looping on all buffers and children... _Originally posted in https://github.com/apache/arrow/pull/40807#discussion_r1611641179_ Authored-by: Matt Topol Signed-off-by: Matt Topol --- cpp/src/arrow/array/data.cc | 12 ++++++++++++ cpp/src/arrow/array/data.h | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/cpp/src/arrow/array/data.cc b/cpp/src/arrow/array/data.cc index 76a43521394c1..83eeb56c496cf 100644 --- a/cpp/src/arrow/array/data.cc +++ b/cpp/src/arrow/array/data.cc @@ -233,28 +233,40 @@ DeviceAllocationType ArrayData::device_type() const { int type = 0; for (const auto& buf : buffers) { if (!buf) continue; +#ifdef NDEBUG + return buf->device_type(); +#else if (type == 0) { type = static_cast(buf->device_type()); } else { DCHECK_EQ(type, static_cast(buf->device_type())); } +#endif } for (const auto& child : child_data) { if (!child) continue; +#ifdef NDEBUG + return child->device_type(); +#else if (type == 0) { type = static_cast(child->device_type()); } else { DCHECK_EQ(type, static_cast(child->device_type())); } +#endif } if (dictionary) { +#ifdef NDEBUG + return dictionary->device_type(); +#else if (type == 0) { type = static_cast(dictionary->device_type()); } else { DCHECK_EQ(type, static_cast(dictionary->device_type())); } +#endif } return type == 0 ? DeviceAllocationType::kCPU : static_cast(type); diff --git a/cpp/src/arrow/array/data.h b/cpp/src/arrow/array/data.h index 0c49f36229a40..e0508fe6980a7 100644 --- a/cpp/src/arrow/array/data.h +++ b/cpp/src/arrow/array/data.h @@ -369,7 +369,7 @@ struct ARROW_EXPORT ArrayData { /// \see GetNullCount int64_t ComputeLogicalNullCount() const; - /// \brief Returns the device_type of the underlying buffers and children + /// \brief Return the device_type of the underlying buffers and children /// /// If there are no buffers in this ArrayData object, it just returns /// DeviceAllocationType::kCPU as a default. We also assume that all buffers From 0b5c53ba0f37b0687ff64b78cbf4f71a11bfdbec Mon Sep 17 00:00:00 2001 From: Laurent Goujon Date: Tue, 28 May 2024 16:41:13 -0700 Subject: [PATCH 22/93] MINOR: [Java] Fix develocity cache directory name in .gitignore (#41866) ### Rationale for this change `.gitignore` is not referencing correctly the develocity cache directory ### What changes are included in this PR? Changing from `.mvn/.develocity.xml` to `.mvn/.develocity/` ### Are these changes tested? No (Checking local git output) ### Are there any user-facing changes? No Authored-by: Laurent Goujon Signed-off-by: David Li --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e2e84fee57e3c..a482f5503c2b9 100644 --- a/.gitignore +++ b/.gitignore @@ -102,4 +102,4 @@ __debug_bin .envrc # Develocity -.mvn/.develocity.xml +.mvn/.develocity/ From 13630c7a8316bccce4a119d193e3a46f5a9a35bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 May 2024 08:44:43 +0900 Subject: [PATCH 23/93] MINOR: [Java] Bump org.apache.commons:commons-compress from 1.26.0 to 1.26.2 in /java (#41853) Bumps org.apache.commons:commons-compress from 1.26.0 to 1.26.2. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.commons:commons-compress&package-manager=maven&previous-version=1.26.0&new-version=1.26.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: David Li --- java/compression/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/compression/pom.xml b/java/compression/pom.xml index 26467dbaf2db3..6ed0be6815ca3 100644 --- a/java/compression/pom.xml +++ b/java/compression/pom.xml @@ -42,7 +42,7 @@ org.apache.commons commons-compress - 1.26.0 + 1.26.2 com.github.luben From 4d524eb40161579c96d1ac11d8e296ab3507f889 Mon Sep 17 00:00:00 2001 From: abandy Date: Tue, 28 May 2024 22:58:22 -0400 Subject: [PATCH 24/93] GH-37938: [Swift] Add initial C data interface implementation (#41342) Continuation for PR: #39091 This add an initial implementation of the C Data interface for swift. During development it was found that null count was not being properly maintained on the arrow buffers and this change is included as well. Also some minor refactoring was done to existing sources to enable this feature. This has been tested from Swift calling into C to import data but not from Swift to C exporting data. Test is currently ongoing. * GitHub Issue: #37938 Authored-by: Alva Bandy Signed-off-by: Sutou Kouhei --- ci/docker/ubuntu-swift.dockerfile | 2 +- dev/release/rat_exclude_files.txt | 1 + swift/.swiftlint.yml | 4 + swift/Arrow/Package.swift | 22 ++- swift/Arrow/Sources/Arrow/ArrowArray.swift | 39 ++-- swift/Arrow/Sources/Arrow/ArrowBuffer.swift | 17 +- .../Arrow/Sources/Arrow/ArrowCExporter.swift | 135 +++++++++++++ .../Arrow/Sources/Arrow/ArrowCImporter.swift | 179 ++++++++++++++++++ .../Sources/Arrow/ArrowReaderHelper.swift | 16 +- swift/Arrow/Sources/Arrow/ArrowSchema.swift | 6 +- swift/Arrow/Sources/Arrow/ArrowTable.swift | 18 +- swift/Arrow/Sources/Arrow/ArrowType.swift | 116 ++++++++++++ swift/Arrow/Sources/Arrow/ChunkedArray.swift | 5 + swift/Arrow/Sources/ArrowC/ArrowCData.c | 30 +++ .../Arrow/Sources/ArrowC/include/ArrowCData.h | 81 ++++++++ swift/Arrow/Tests/ArrowTests/CDataTests.swift | 125 ++++++++++++ swift/Arrow/Tests/ArrowTests/IPCTests.swift | 16 +- .../Tests/ArrowTests/RecordBatchTests.swift | 4 +- swift/Arrow/Tests/ArrowTests/TableTests.swift | 4 +- swift/ArrowFlight/Package.swift | 4 +- .../Tests/ArrowFlightTests/FlightTest.swift | 6 +- swift/CDataWGo/.gitignore | 8 + swift/CDataWGo/Package.swift | 43 +++++ .../CDataWGo/Sources/go-swift/CDataTest.swift | 132 +++++++++++++ swift/CDataWGo/go.mod | 41 ++++ swift/CDataWGo/go.sum | 75 ++++++++ swift/CDataWGo/include/go_swift.h | 30 +++ swift/CDataWGo/main.go | 127 +++++++++++++ 28 files changed, 1231 insertions(+), 55 deletions(-) create mode 100644 swift/Arrow/Sources/Arrow/ArrowCExporter.swift create mode 100644 swift/Arrow/Sources/Arrow/ArrowCImporter.swift create mode 100644 swift/Arrow/Sources/ArrowC/ArrowCData.c create mode 100644 swift/Arrow/Sources/ArrowC/include/ArrowCData.h create mode 100644 swift/Arrow/Tests/ArrowTests/CDataTests.swift create mode 100644 swift/CDataWGo/.gitignore create mode 100644 swift/CDataWGo/Package.swift create mode 100644 swift/CDataWGo/Sources/go-swift/CDataTest.swift create mode 100644 swift/CDataWGo/go.mod create mode 100644 swift/CDataWGo/go.sum create mode 100644 swift/CDataWGo/include/go_swift.h create mode 100644 swift/CDataWGo/main.go diff --git a/ci/docker/ubuntu-swift.dockerfile b/ci/docker/ubuntu-swift.dockerfile index 4789c9188c226..26950b806d1bc 100644 --- a/ci/docker/ubuntu-swift.dockerfile +++ b/ci/docker/ubuntu-swift.dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM swift:5.7.3 +FROM swift:5.9.0 # Go is needed for generating test data RUN apt-get update -y -q && \ diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index f4d7b411c4dc2..0cc1348f50b95 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -150,3 +150,4 @@ r/tools/nixlibs-allowlist.txt ruby/red-arrow/.yardopts .github/pull_request_template.md swift/data-generator/swift-datagen/go.sum +swift/CDataWGo/go.sum diff --git a/swift/.swiftlint.yml b/swift/.swiftlint.yml index d447bf9d5d97c..7e4da29f3741c 100644 --- a/swift/.swiftlint.yml +++ b/swift/.swiftlint.yml @@ -16,10 +16,14 @@ # under the License. included: + - Arrow/Package.swift - Arrow/Sources - Arrow/Tests + - ArrowFlight/Package.swift - ArrowFlight/Sources - ArrowFlight/Tests + - CDataWGo/Package.swift + - CDataWGo/Sources/go-swift excluded: - Arrow/Sources/Arrow/File_generated.swift - Arrow/Sources/Arrow/Message_generated.swift diff --git a/swift/Arrow/Package.swift b/swift/Arrow/Package.swift index 946eb999c798a..6f19136fd4292 100644 --- a/swift/Arrow/Package.swift +++ b/swift/Arrow/Package.swift @@ -26,28 +26,34 @@ let package = Package( .macOS(.v10_14) ], products: [ - // Products define the executables and libraries a package produces, and make them visible to other packages. .library( name: "Arrow", - targets: ["Arrow"]), + targets: ["Arrow"]) ], dependencies: [ // The latest version of flatbuffers v23.5.26 was built in May 26, 2023 // and therefore doesn't include the unaligned buffer swift changes. // This can be changed back to using the tag once a new version of // flatbuffers has been released. - .package(url: "https://github.com/google/flatbuffers.git", branch: "master") + .package(url: "https://github.com/google/flatbuffers.git", branch: "master"), + .package( + url: "https://github.com/apple/swift-atomics.git", + .upToNextMajor(from: "1.2.0") // or `.upToNextMinor + ) ], targets: [ - // Targets are the basic building blocks of a package. A target can define a module or a test suite. - // Targets can depend on other targets in this package, and on products in packages this package depends on. + .target( + name: "ArrowC", + path: "Sources/ArrowC" + ), .target( name: "Arrow", - dependencies: [ - .product(name: "FlatBuffers", package: "flatbuffers") + dependencies: ["ArrowC", + .product(name: "FlatBuffers", package: "flatbuffers"), + .product(name: "Atomics", package: "swift-atomics") ]), .testTarget( name: "ArrowTests", - dependencies: ["Arrow"]), + dependencies: ["Arrow", "ArrowC"]) ] ) diff --git a/swift/Arrow/Sources/Arrow/ArrowArray.swift b/swift/Arrow/Sources/Arrow/ArrowArray.swift index 88b43e63a92b7..32b6ba1704511 100644 --- a/swift/Arrow/Sources/Arrow/ArrowArray.swift +++ b/swift/Arrow/Sources/Arrow/ArrowArray.swift @@ -17,16 +17,29 @@ import Foundation -public class ArrowArrayHolder { +public protocol ArrowArrayHolder { + var type: ArrowType {get} + var length: UInt {get} + var nullCount: UInt {get} + var array: Any {get} + var data: ArrowData {get} + var getBufferData: () -> [Data] {get} + var getBufferDataSizes: () -> [Int] {get} + var getArrowColumn: (ArrowField, [ArrowArrayHolder]) throws -> ArrowColumn {get} +} + +public class ArrowArrayHolderImpl: ArrowArrayHolder { + public let array: Any + public let data: ArrowData public let type: ArrowType public let length: UInt public let nullCount: UInt - public let array: Any public let getBufferData: () -> [Data] public let getBufferDataSizes: () -> [Int] - private let getArrowColumn: (ArrowField, [ArrowArrayHolder]) throws -> ArrowColumn + public let getArrowColumn: (ArrowField, [ArrowArrayHolder]) throws -> ArrowColumn public init(_ arrowArray: ArrowArray) { self.array = arrowArray + self.data = arrowArray.arrowData self.length = arrowArray.length self.type = arrowArray.arrowData.type self.nullCount = arrowArray.nullCount @@ -60,19 +73,9 @@ public class ArrowArrayHolder { return ArrowColumn(field, chunked: ChunkedArrayHolder(try ChunkedArray(arrays))) } } - - public static func makeArrowColumn(_ field: ArrowField, - holders: [ArrowArrayHolder] - ) -> Result { - do { - return .success(try holders[0].getArrowColumn(field, holders)) - } catch { - return .failure(.runtimeError("\(error)")) - } - } } -public class ArrowArray: AsString { +public class ArrowArray: AsString, AnyArray { public typealias ItemType = T public let arrowData: ArrowData public var nullCount: UInt {return self.arrowData.nullCount} @@ -101,6 +104,14 @@ public class ArrowArray: AsString { return "\(self[index]!)" } + + public func asAny(_ index: UInt) -> Any? { + if self[index] == nil { + return nil + } + + return self[index]! + } } public class FixedArray: ArrowArray { diff --git a/swift/Arrow/Sources/Arrow/ArrowBuffer.swift b/swift/Arrow/Sources/Arrow/ArrowBuffer.swift index 4ac4eb93c91db..1ff53cd7dd5d9 100644 --- a/swift/Arrow/Sources/Arrow/ArrowBuffer.swift +++ b/swift/Arrow/Sources/Arrow/ArrowBuffer.swift @@ -22,16 +22,20 @@ public class ArrowBuffer { static let maxLength = UInt.max fileprivate(set) var length: UInt let capacity: UInt - let rawPointer: UnsafeMutableRawPointer + public let rawPointer: UnsafeMutableRawPointer + let isMemoryOwner: Bool - init(length: UInt, capacity: UInt, rawPointer: UnsafeMutableRawPointer) { + init(length: UInt, capacity: UInt, rawPointer: UnsafeMutableRawPointer, isMemoryOwner: Bool = true) { self.length = length self.capacity = capacity self.rawPointer = rawPointer + self.isMemoryOwner = isMemoryOwner } deinit { - self.rawPointer.deallocate() + if isMemoryOwner { + self.rawPointer.deallocate() + } } func append(to data: inout Data) { @@ -39,6 +43,13 @@ public class ArrowBuffer { data.append(ptr, count: Int(capacity)) } + static func createEmptyBuffer() -> ArrowBuffer { + return ArrowBuffer( + length: 0, + capacity: 0, + rawPointer: UnsafeMutableRawPointer.allocate(byteCount: 0, alignment: .zero)) + } + static func createBuffer(_ data: [UInt8], length: UInt) -> ArrowBuffer { let byteCount = UInt(data.count) let capacity = alignTo64(byteCount) diff --git a/swift/Arrow/Sources/Arrow/ArrowCExporter.swift b/swift/Arrow/Sources/Arrow/ArrowCExporter.swift new file mode 100644 index 0000000000000..aa93f0cb7e389 --- /dev/null +++ b/swift/Arrow/Sources/Arrow/ArrowCExporter.swift @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import ArrowC +import Atomics + +// The memory used by UnsafeAtomic is not automatically +// reclaimed. Since this value is initialized once +// and used until the program/app is closed it's +// memory will be released on program/app exit +let exportDataCounter: UnsafeAtomic = .create(0) + +public class ArrowCExporter { + private class ExportData { + let id: Int + init() { + id = exportDataCounter.loadThenWrappingIncrement(ordering: .relaxed) + ArrowCExporter.exportedData[id] = self + } + } + + private class ExportSchema: ExportData { + public let arrowTypeNameCstr: UnsafePointer + public let nameCstr: UnsafePointer + private let arrowType: ArrowType + private let name: String + private let arrowTypeName: String + init(_ arrowType: ArrowType, name: String = "") throws { + self.arrowType = arrowType + // keeping the name str to ensure the cstring buffer remains valid + self.name = name + self.arrowTypeName = try arrowType.cDataFormatId + self.nameCstr = (self.name as NSString).utf8String! + self.arrowTypeNameCstr = (self.arrowTypeName as NSString).utf8String! + super.init() + } + } + + private class ExportArray: ExportData { + private let arrowData: ArrowData + private(set) var data = [UnsafeRawPointer?]() + private(set) var buffers: UnsafeMutablePointer + init(_ arrowData: ArrowData) { + // keep a reference to the ArrowData + // obj so the memory doesn't get + // deallocated + self.arrowData = arrowData + for arrowBuffer in arrowData.buffers { + data.append(arrowBuffer.rawPointer) + } + + self.buffers = UnsafeMutablePointer(mutating: data) + super.init() + } + } + + private static var exportedData = [Int: ExportData]() + public init() {} + + public func exportType(_ cSchema: inout ArrowC.ArrowSchema, arrowType: ArrowType, name: String = "") -> + Result { + do { + let exportSchema = try ExportSchema(arrowType, name: name) + cSchema.format = exportSchema.arrowTypeNameCstr + cSchema.name = exportSchema.nameCstr + cSchema.private_data = + UnsafeMutableRawPointer(mutating: UnsafeRawPointer(bitPattern: exportSchema.id)) + cSchema.release = {(data: UnsafeMutablePointer?) in + let arraySchema = data!.pointee + let exportId = Int(bitPattern: arraySchema.private_data) + guard ArrowCExporter.exportedData[exportId] != nil else { + fatalError("Export schema not found with id \(exportId)") + } + + // the data associated with this exportSchema object + // which includes the C strings for the format and name + // be deallocated upon removal + ArrowCExporter.exportedData.removeValue(forKey: exportId) + ArrowC.ArrowSwiftClearReleaseSchema(data) + } + } catch { + return .failure(.unknownError("\(error)")) + } + return .success(true) + } + + public func exportField(_ schema: inout ArrowC.ArrowSchema, field: ArrowField) -> + Result { + return exportType(&schema, arrowType: field.type, name: field.name) + } + + public func exportArray(_ cArray: inout ArrowC.ArrowArray, arrowData: ArrowData) { + let exportArray = ExportArray(arrowData) + cArray.buffers = exportArray.buffers + cArray.length = Int64(arrowData.length) + cArray.null_count = Int64(arrowData.nullCount) + cArray.n_buffers = Int64(arrowData.buffers.count) + // Swift Arrow does not currently support children or dictionaries + // This will need to be updated once support has been added + cArray.n_children = 0 + cArray.children = nil + cArray.dictionary = nil + cArray.private_data = + UnsafeMutableRawPointer(mutating: UnsafeRawPointer(bitPattern: exportArray.id)) + cArray.release = {(data: UnsafeMutablePointer?) in + let arrayData = data!.pointee + let exportId = Int(bitPattern: arrayData.private_data) + guard ArrowCExporter.exportedData[exportId] != nil else { + fatalError("Export data not found with id \(exportId)") + } + + // the data associated with this exportArray object + // which includes the entire arrowData object + // and the buffers UnsafeMutablePointer[] will + // be deallocated upon removal + ArrowCExporter.exportedData.removeValue(forKey: exportId) + ArrowC.ArrowSwiftClearReleaseArray(data) + } + } +} diff --git a/swift/Arrow/Sources/Arrow/ArrowCImporter.swift b/swift/Arrow/Sources/Arrow/ArrowCImporter.swift new file mode 100644 index 0000000000000..8a4cf47fc0b43 --- /dev/null +++ b/swift/Arrow/Sources/Arrow/ArrowCImporter.swift @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import ArrowC + +public class ImportArrayHolder: ArrowArrayHolder { + let cArrayPtr: UnsafePointer + public var type: ArrowType {self.holder.type} + public var length: UInt {self.holder.length} + public var nullCount: UInt {self.holder.nullCount} + public var array: Any {self.holder.array} + public var data: ArrowData {self.holder.data} + public var getBufferData: () -> [Data] {self.holder.getBufferData} + public var getBufferDataSizes: () -> [Int] {self.holder.getBufferDataSizes} + public var getArrowColumn: (ArrowField, [ArrowArrayHolder]) throws -> ArrowColumn {self.holder.getArrowColumn} + private let holder: ArrowArrayHolder + init(_ holder: ArrowArrayHolder, cArrayPtr: UnsafePointer) { + self.cArrayPtr = cArrayPtr + self.holder = holder + } + + deinit { + if self.cArrayPtr.pointee.release != nil { + ArrowCImporter.release(self.cArrayPtr) + } + } +} + +public class ArrowCImporter { + private func appendToBuffer( + _ cBuffer: UnsafeRawPointer?, + arrowBuffers: inout [ArrowBuffer], + length: UInt) { + if cBuffer == nil { + arrowBuffers.append(ArrowBuffer.createEmptyBuffer()) + return + } + + let pointer = UnsafeMutableRawPointer(mutating: cBuffer)! + arrowBuffers.append( + ArrowBuffer(length: length, capacity: length, rawPointer: pointer, isMemoryOwner: false)) + } + + public init() {} + + public func importType(_ cArrow: String, name: String = "") -> + Result { + do { + let type = try ArrowType.fromCDataFormatId(cArrow) + return .success(ArrowField(name, type: ArrowType(type.info), isNullable: true)) + } catch { + return .failure(.invalid("Error occurred while attempting to import type: \(error)")) + } + } + + public func importField(_ cSchema: ArrowC.ArrowSchema) -> + Result { + if cSchema.n_children > 0 { + ArrowCImporter.release(cSchema) + return .failure(.invalid("Children currently not supported")) + } else if cSchema.dictionary != nil { + ArrowCImporter.release(cSchema) + return .failure(.invalid("Dictinoary types currently not supported")) + } + + switch importType( + String(cString: cSchema.format), name: String(cString: cSchema.name)) { + case .success(let field): + ArrowCImporter.release(cSchema) + return .success(field) + case .failure(let err): + ArrowCImporter.release(cSchema) + return .failure(err) + } + } + + public func importArray( + _ cArray: UnsafePointer, + arrowType: ArrowType, + isNullable: Bool = false + ) -> Result { + let arrowField = ArrowField("", type: arrowType, isNullable: isNullable) + return importArray(cArray, arrowField: arrowField) + } + + public func importArray( // swiftlint:disable:this cyclomatic_complexity function_body_length + _ cArrayPtr: UnsafePointer, + arrowField: ArrowField + ) -> Result { + let cArray = cArrayPtr.pointee + if cArray.null_count < 0 { + ArrowCImporter.release(cArrayPtr) + return .failure(.invalid("Uncomputed null count is not supported")) + } else if cArray.n_children > 0 { + ArrowCImporter.release(cArrayPtr) + return .failure(.invalid("Children currently not supported")) + } else if cArray.dictionary != nil { + ArrowCImporter.release(cArrayPtr) + return .failure(.invalid("Dictionary types currently not supported")) + } else if cArray.offset != 0 { + ArrowCImporter.release(cArrayPtr) + return .failure(.invalid("Offset of 0 is required but found offset: \(cArray.offset)")) + } + + let arrowType = arrowField.type + let length = UInt(cArray.length) + let nullCount = UInt(cArray.null_count) + var arrowBuffers = [ArrowBuffer]() + + if cArray.n_buffers > 0 { + if cArray.buffers == nil { + ArrowCImporter.release(cArrayPtr) + return .failure(.invalid("C array buffers is nil")) + } + + switch arrowType.info { + case .variableInfo: + if cArray.n_buffers != 3 { + ArrowCImporter.release(cArrayPtr) + return .failure( + .invalid("Variable buffer count expected 3 but found \(cArray.n_buffers)")) + } + + appendToBuffer(cArray.buffers[0], arrowBuffers: &arrowBuffers, length: UInt(ceil(Double(length) / 8))) + appendToBuffer(cArray.buffers[1], arrowBuffers: &arrowBuffers, length: length) + let lastOffsetLength = cArray.buffers[1]! + .advanced(by: Int(length) * MemoryLayout.stride) + .load(as: Int32.self) + appendToBuffer(cArray.buffers[2], arrowBuffers: &arrowBuffers, length: UInt(lastOffsetLength)) + default: + if cArray.n_buffers != 2 { + ArrowCImporter.release(cArrayPtr) + return .failure(.invalid("Expected buffer count 2 but found \(cArray.n_buffers)")) + } + + appendToBuffer(cArray.buffers[0], arrowBuffers: &arrowBuffers, length: UInt(ceil(Double(length) / 8))) + appendToBuffer(cArray.buffers[1], arrowBuffers: &arrowBuffers, length: length) + } + } + + switch makeArrayHolder(arrowField, buffers: arrowBuffers, nullCount: nullCount) { + case .success(let holder): + return .success(ImportArrayHolder(holder, cArrayPtr: cArrayPtr)) + case .failure(let err): + ArrowCImporter.release(cArrayPtr) + return .failure(err) + } + } + + public static func release(_ cArrayPtr: UnsafePointer) { + if cArrayPtr.pointee.release != nil { + let cSchemaMutablePtr = UnsafeMutablePointer(mutating: cArrayPtr) + cArrayPtr.pointee.release(cSchemaMutablePtr) + } + } + + public static func release(_ cSchema: ArrowC.ArrowSchema) { + if cSchema.release != nil { + let cSchemaPtr = UnsafeMutablePointer.allocate(capacity: 1) + cSchemaPtr.initialize(to: cSchema) + cSchema.release(cSchemaPtr) + } + } +} diff --git a/swift/Arrow/Sources/Arrow/ArrowReaderHelper.swift b/swift/Arrow/Sources/Arrow/ArrowReaderHelper.swift index fb4a13b766f10..c701653ecb2c9 100644 --- a/swift/Arrow/Sources/Arrow/ArrowReaderHelper.swift +++ b/swift/Arrow/Sources/Arrow/ArrowReaderHelper.swift @@ -23,7 +23,7 @@ private func makeBinaryHolder(_ buffers: [ArrowBuffer], do { let arrowType = ArrowType(ArrowType.ArrowBinary) let arrowData = try ArrowData(arrowType, buffers: buffers, nullCount: nullCount) - return .success(ArrowArrayHolder(BinaryArray(arrowData))) + return .success(ArrowArrayHolderImpl(BinaryArray(arrowData))) } catch let error as ArrowError { return .failure(error) } catch { @@ -36,7 +36,7 @@ private func makeStringHolder(_ buffers: [ArrowBuffer], do { let arrowType = ArrowType(ArrowType.ArrowString) let arrowData = try ArrowData(arrowType, buffers: buffers, nullCount: nullCount) - return .success(ArrowArrayHolder(StringArray(arrowData))) + return .success(ArrowArrayHolderImpl(StringArray(arrowData))) } catch let error as ArrowError { return .failure(error) } catch { @@ -51,11 +51,11 @@ private func makeDateHolder(_ field: ArrowField, do { if field.type.id == .date32 { let arrowData = try ArrowData(field.type, buffers: buffers, nullCount: nullCount) - return .success(ArrowArrayHolder(Date32Array(arrowData))) + return .success(ArrowArrayHolderImpl(Date32Array(arrowData))) } let arrowData = try ArrowData(field.type, buffers: buffers, nullCount: nullCount) - return .success(ArrowArrayHolder(Date64Array(arrowData))) + return .success(ArrowArrayHolderImpl(Date64Array(arrowData))) } catch let error as ArrowError { return .failure(error) } catch { @@ -71,7 +71,7 @@ private func makeTimeHolder(_ field: ArrowField, if field.type.id == .time32 { if let arrowType = field.type as? ArrowTypeTime32 { let arrowData = try ArrowData(arrowType, buffers: buffers, nullCount: nullCount) - return .success(ArrowArrayHolder(FixedArray(arrowData))) + return .success(ArrowArrayHolderImpl(FixedArray(arrowData))) } else { return .failure(.invalid("Incorrect field type for time: \(field.type)")) } @@ -79,7 +79,7 @@ private func makeTimeHolder(_ field: ArrowField, if let arrowType = field.type as? ArrowTypeTime64 { let arrowData = try ArrowData(arrowType, buffers: buffers, nullCount: nullCount) - return .success(ArrowArrayHolder(FixedArray(arrowData))) + return .success(ArrowArrayHolderImpl(FixedArray(arrowData))) } else { return .failure(.invalid("Incorrect field type for time: \(field.type)")) } @@ -95,7 +95,7 @@ private func makeBoolHolder(_ buffers: [ArrowBuffer], do { let arrowType = ArrowType(ArrowType.ArrowBool) let arrowData = try ArrowData(arrowType, buffers: buffers, nullCount: nullCount) - return .success(ArrowArrayHolder(BoolArray(arrowData))) + return .success(ArrowArrayHolderImpl(BoolArray(arrowData))) } catch let error as ArrowError { return .failure(error) } catch { @@ -109,7 +109,7 @@ private func makeFixedHolder( ) -> Result { do { let arrowData = try ArrowData(field.type, buffers: buffers, nullCount: nullCount) - return .success(ArrowArrayHolder(FixedArray(arrowData))) + return .success(ArrowArrayHolderImpl(FixedArray(arrowData))) } catch let error as ArrowError { return .failure(error) } catch { diff --git a/swift/Arrow/Sources/Arrow/ArrowSchema.swift b/swift/Arrow/Sources/Arrow/ArrowSchema.swift index 45f13a1551c3d..65c506d51cdd6 100644 --- a/swift/Arrow/Sources/Arrow/ArrowSchema.swift +++ b/swift/Arrow/Sources/Arrow/ArrowSchema.swift @@ -17,9 +17,9 @@ import Foundation public class ArrowField { - let type: ArrowType - let name: String - let isNullable: Bool + public let type: ArrowType + public let name: String + public let isNullable: Bool init(_ name: String, type: ArrowType, isNullable: Bool) { self.name = name diff --git a/swift/Arrow/Sources/Arrow/ArrowTable.swift b/swift/Arrow/Sources/Arrow/ArrowTable.swift index 7677fb4f33a19..b9d15154c4f94 100644 --- a/swift/Arrow/Sources/Arrow/ArrowTable.swift +++ b/swift/Arrow/Sources/Arrow/ArrowTable.swift @@ -64,7 +64,7 @@ public class ArrowTable { let builder = ArrowTable.Builder() for index in 0.. Result { + do { + return .success(try holders[0].getArrowColumn(field, holders)) + } catch { + return .failure(.runtimeError("\(error)")) + } + } + public class Builder { let schemaBuilder = ArrowSchema.Builder() var columns = [ArrowColumn]() @@ -172,6 +183,11 @@ public class RecordBatch { return (arrayHolder.array as! ArrowArray) // swiftlint:disable:this force_cast } + public func anyData(for columnIndex: Int) -> AnyArray { + let arrayHolder = column(columnIndex) + return (arrayHolder.array as! AnyArray) // swiftlint:disable:this force_cast + } + public func column(_ index: Int) -> ArrowArrayHolder { return self.columns[index] } diff --git a/swift/Arrow/Sources/Arrow/ArrowType.swift b/swift/Arrow/Sources/Arrow/ArrowType.swift index f5a869f7cdaff..e1ada4b9734ea 100644 --- a/swift/Arrow/Sources/Arrow/ArrowType.swift +++ b/swift/Arrow/Sources/Arrow/ArrowType.swift @@ -90,6 +90,17 @@ public class ArrowTypeTime32: ArrowType { self.unit = unit super.init(ArrowType.ArrowTime32) } + + public override var cDataFormatId: String { + get throws { + switch self.unit { + case .milliseconds: + return "ttm" + case .seconds: + return "tts" + } + } + } } public class ArrowTypeTime64: ArrowType { @@ -98,6 +109,17 @@ public class ArrowTypeTime64: ArrowType { self.unit = unit super.init(ArrowType.ArrowTime64) } + + public override var cDataFormatId: String { + get throws { + switch self.unit { + case .microseconds: + return "ttu" + case .nanoseconds: + return "ttn" + } + } + } } public class ArrowType { @@ -209,6 +231,100 @@ public class ArrowType { fatalError("Stride requested for unknown type: \(self)") } } + + public var cDataFormatId: String { + get throws { + switch self.id { + case ArrowTypeId.int8: + return "c" + case ArrowTypeId.int16: + return "s" + case ArrowTypeId.int32: + return "i" + case ArrowTypeId.int64: + return "l" + case ArrowTypeId.uint8: + return "C" + case ArrowTypeId.uint16: + return "S" + case ArrowTypeId.uint32: + return "I" + case ArrowTypeId.uint64: + return "L" + case ArrowTypeId.float: + return "f" + case ArrowTypeId.double: + return "g" + case ArrowTypeId.boolean: + return "b" + case ArrowTypeId.date32: + return "tdD" + case ArrowTypeId.date64: + return "tdm" + case ArrowTypeId.time32: + if let time32 = self as? ArrowTypeTime32 { + return try time32.cDataFormatId + } + return "tts" + case ArrowTypeId.time64: + if let time64 = self as? ArrowTypeTime64 { + return try time64.cDataFormatId + } + return "ttu" + case ArrowTypeId.binary: + return "z" + case ArrowTypeId.string: + return "u" + default: + throw ArrowError.notImplemented + } + } + } + + public static func fromCDataFormatId( // swiftlint:disable:this cyclomatic_complexity + _ from: String) throws -> ArrowType { + if from == "c" { + return ArrowType(ArrowType.ArrowInt8) + } else if from == "s" { + return ArrowType(ArrowType.ArrowInt16) + } else if from == "i" { + return ArrowType(ArrowType.ArrowInt32) + } else if from == "l" { + return ArrowType(ArrowType.ArrowInt64) + } else if from == "C" { + return ArrowType(ArrowType.ArrowUInt8) + } else if from == "S" { + return ArrowType(ArrowType.ArrowUInt16) + } else if from == "I" { + return ArrowType(ArrowType.ArrowUInt32) + } else if from == "L" { + return ArrowType(ArrowType.ArrowUInt64) + } else if from == "f" { + return ArrowType(ArrowType.ArrowFloat) + } else if from == "g" { + return ArrowType(ArrowType.ArrowDouble) + } else if from == "b" { + return ArrowType(ArrowType.ArrowBool) + } else if from == "tdD" { + return ArrowType(ArrowType.ArrowDate32) + } else if from == "tdm" { + return ArrowType(ArrowType.ArrowDate64) + } else if from == "tts" { + return ArrowTypeTime32(.seconds) + } else if from == "ttm" { + return ArrowTypeTime32(.milliseconds) + } else if from == "ttu" { + return ArrowTypeTime64(.microseconds) + } else if from == "ttn" { + return ArrowTypeTime64(.nanoseconds) + } else if from == "z" { + return ArrowType(ArrowType.ArrowBinary) + } else if from == "u" { + return ArrowType(ArrowType.ArrowString) + } + + throw ArrowError.notImplemented + } } extension ArrowType.Info: Equatable { diff --git a/swift/Arrow/Sources/Arrow/ChunkedArray.swift b/swift/Arrow/Sources/Arrow/ChunkedArray.swift index 3a06aa46550df..c5ccfe4aec0e6 100644 --- a/swift/Arrow/Sources/Arrow/ChunkedArray.swift +++ b/swift/Arrow/Sources/Arrow/ChunkedArray.swift @@ -17,6 +17,11 @@ import Foundation +public protocol AnyArray { + func asAny(_ index: UInt) -> Any? + var length: UInt {get} +} + public protocol AsString { func asString(_ index: UInt) -> String } diff --git a/swift/Arrow/Sources/ArrowC/ArrowCData.c b/swift/Arrow/Sources/ArrowC/ArrowCData.c new file mode 100644 index 0000000000000..ac366febdaed8 --- /dev/null +++ b/swift/Arrow/Sources/ArrowC/ArrowCData.c @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "include/ArrowCData.h" + +void ArrowSwiftClearReleaseSchema(struct ArrowSchema* arrowSchema) { + if(arrowSchema) { + arrowSchema->release = NULL; + } +} + +void ArrowSwiftClearReleaseArray(struct ArrowArray* arrowArray) { + if(arrowArray) { + arrowArray->release = NULL; + } +} diff --git a/swift/Arrow/Sources/ArrowC/include/ArrowCData.h b/swift/Arrow/Sources/ArrowC/include/ArrowCData.h new file mode 100644 index 0000000000000..9df8992114be3 --- /dev/null +++ b/swift/Arrow/Sources/ArrowC/include/ArrowCData.h @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef ARROW_C_DATA_INTERFACE +#define ARROW_C_DATA_INTERFACE + +#define ARROW_FLAG_DICTIONARY_ORDERED 1 +#define ARROW_FLAG_NULLABLE 2 +#define ARROW_FLAG_MAP_KEYS_SORTED 4 + +#include +#include +#include // Must have this! + + +#ifdef __cplusplus +extern "C" { +#endif + +struct ArrowSchema { + // Array type description + const char* format; + const char* name; + const char* metadata; + int64_t flags; + int64_t n_children; + struct ArrowSchema** children; + struct ArrowSchema* dictionary; + + // Release callback + void (*release)(struct ArrowSchema*); + // Opaque producer-specific data + void* private_data; +}; + +struct ArrowArray { + // Array data description + int64_t length; + int64_t null_count; + int64_t offset; + int64_t n_buffers; + int64_t n_children; + const void** buffers; + struct ArrowArray** children; + struct ArrowArray* dictionary; + + // Release callback + void (*release)(struct ArrowArray*); + // Opaque producer-specific data + void* private_data; +}; + +// Not able to set the release on the schema +// to NULL in Swift. nil in Swift is not +// equivalent to NULL. +void ArrowSwiftClearReleaseSchema(struct ArrowSchema*); + +// Not able to set the release on the array +// to NULL in Swift. nil in Swift is not +// equivalent to NULL. +void ArrowSwiftClearReleaseArray(struct ArrowArray*); + +#ifdef __cplusplus +} +#endif + +#endif // ARROW_C_DATA_INTERFACE diff --git a/swift/Arrow/Tests/ArrowTests/CDataTests.swift b/swift/Arrow/Tests/ArrowTests/CDataTests.swift new file mode 100644 index 0000000000000..2344b234745a2 --- /dev/null +++ b/swift/Arrow/Tests/ArrowTests/CDataTests.swift @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import XCTest +@testable import Arrow +import ArrowC + +final class CDataTests: XCTestCase { + func makeSchema() -> Arrow.ArrowSchema { + let schemaBuilder = ArrowSchema.Builder() + return schemaBuilder + .addField("colBool", type: ArrowType(ArrowType.ArrowBool), isNullable: false) + .addField("colUInt8", type: ArrowType(ArrowType.ArrowUInt8), isNullable: true) + .addField("colUInt16", type: ArrowType(ArrowType.ArrowUInt16), isNullable: true) + .addField("colUInt32", type: ArrowType(ArrowType.ArrowUInt32), isNullable: true) + .addField("colUInt64", type: ArrowType(ArrowType.ArrowUInt64), isNullable: true) + .addField("colInt8", type: ArrowType(ArrowType.ArrowInt8), isNullable: false) + .addField("colInt16", type: ArrowType(ArrowType.ArrowInt16), isNullable: false) + .addField("colInt32", type: ArrowType(ArrowType.ArrowInt32), isNullable: false) + .addField("colInt64", type: ArrowType(ArrowType.ArrowInt64), isNullable: false) + .addField("colString", type: ArrowType(ArrowType.ArrowString), isNullable: false) + .addField("colBinary", type: ArrowType(ArrowType.ArrowBinary), isNullable: false) + .addField("colDate32", type: ArrowType(ArrowType.ArrowDate32), isNullable: false) + .addField("colDate64", type: ArrowType(ArrowType.ArrowDate64), isNullable: false) + .addField("colTime32", type: ArrowType(ArrowType.ArrowTime32), isNullable: false) + .addField("colTime32s", type: ArrowTypeTime32(.seconds), isNullable: false) + .addField("colTime32m", type: ArrowTypeTime32(.milliseconds), isNullable: false) + .addField("colTime64", type: ArrowType(ArrowType.ArrowTime64), isNullable: false) + .addField("colTime64u", type: ArrowTypeTime64(.microseconds), isNullable: false) + .addField("colTime64n", type: ArrowTypeTime64(.nanoseconds), isNullable: false) + .addField("colTime64", type: ArrowType(ArrowType.ArrowTime64), isNullable: false) + .addField("colFloat", type: ArrowType(ArrowType.ArrowFloat), isNullable: false) + .addField("colDouble", type: ArrowType(ArrowType.ArrowDouble), isNullable: false) + .finish() + } + + func checkImportField(_ cSchema: ArrowC.ArrowSchema, name: String, type: ArrowType.Info) throws { + let importer = ArrowCImporter() + switch importer.importField(cSchema) { + case .success(let arrowField): + XCTAssertEqual(arrowField.type.info, type) + XCTAssertEqual(arrowField.name, name) + case .failure(let error): + throw error + } + } + + func testImportExportSchema() throws { + let schema = makeSchema() + let exporter = ArrowCExporter() + for arrowField in schema.fields { + var cSchema = ArrowC.ArrowSchema() + switch exporter.exportField(&cSchema, field: arrowField) { + case .success: + try checkImportField(cSchema, name: arrowField.name, type: arrowField.type.info) + case .failure(let error): + throw error + } + } + } + + func testImportExportArray() throws { + let stringBuilder = try ArrowArrayBuilders.loadStringArrayBuilder() + for index in 0..<100 { + if index % 10 == 9 { + stringBuilder.append(nil) + } else { + stringBuilder.append("test" + String(index)) + } + } + + XCTAssertEqual(stringBuilder.nullCount, 10) + XCTAssertEqual(stringBuilder.length, 100) + XCTAssertEqual(stringBuilder.capacity, 648) + let stringArray = try stringBuilder.finish() + let exporter = ArrowCExporter() + var cArray = ArrowC.ArrowArray() + exporter.exportArray(&cArray, arrowData: stringArray.arrowData) + let cArrayMutPtr = UnsafeMutablePointer.allocate(capacity: 1) + cArrayMutPtr.pointee = cArray + defer { + cArrayMutPtr.deallocate() + } + + let importer = ArrowCImporter() + switch importer.importArray(UnsafePointer(cArrayMutPtr), arrowType: ArrowType(ArrowType.ArrowString)) { + case .success(let holder): + let builder = RecordBatch.Builder() + switch builder + .addColumn("test", arrowArray: holder) + .finish() { + case .success(let rb): + XCTAssertEqual(rb.columnCount, 1) + XCTAssertEqual(rb.length, 100) + let col1: Arrow.ArrowArray = rb.data(for: 0) + for index in 0.. RecordBatch { floatBuilder.append(433.334) floatBuilder.append(544.445) - let uint8Holder = ArrowArrayHolder(try uint8Builder.finish()) - let stringHolder = ArrowArrayHolder(try stringBuilder.finish()) - let date32Holder = ArrowArrayHolder(try date32Builder.finish()) - let int32Holder = ArrowArrayHolder(try int32Builder.finish()) - let floatHolder = ArrowArrayHolder(try floatBuilder.finish()) + let uint8Holder = ArrowArrayHolderImpl(try uint8Builder.finish()) + let stringHolder = ArrowArrayHolderImpl(try stringBuilder.finish()) + let date32Holder = ArrowArrayHolderImpl(try date32Builder.finish()) + let int32Holder = ArrowArrayHolderImpl(try int32Builder.finish()) + let floatHolder = ArrowArrayHolderImpl(try floatBuilder.finish()) let result = RecordBatch.Builder() .addColumn("col1", arrowArray: uint8Holder) .addColumn("col2", arrowArray: stringHolder) @@ -279,7 +279,7 @@ final class IPCFileReaderTests: XCTestCase { binaryBuilder.append("test33".data(using: .utf8)) binaryBuilder.append("test44".data(using: .utf8)) - let binaryHolder = ArrowArrayHolder(try binaryBuilder.finish()) + let binaryHolder = ArrowArrayHolderImpl(try binaryBuilder.finish()) let result = RecordBatch.Builder() .addColumn("binary", arrowArray: binaryHolder) .finish() @@ -307,8 +307,8 @@ final class IPCFileReaderTests: XCTestCase { time32Builder.append(2) time32Builder.append(nil) time32Builder.append(3) - let time64Holder = ArrowArrayHolder(try time64Builder.finish()) - let time32Holder = ArrowArrayHolder(try time32Builder.finish()) + let time64Holder = ArrowArrayHolderImpl(try time64Builder.finish()) + let time32Holder = ArrowArrayHolderImpl(try time32Builder.finish()) let result = RecordBatch.Builder() .addColumn("time64", arrowArray: time64Holder) .addColumn("time32", arrowArray: time32Holder) diff --git a/swift/Arrow/Tests/ArrowTests/RecordBatchTests.swift b/swift/Arrow/Tests/ArrowTests/RecordBatchTests.swift index 8820f1cdb1a91..9961781f30833 100644 --- a/swift/Arrow/Tests/ArrowTests/RecordBatchTests.swift +++ b/swift/Arrow/Tests/ArrowTests/RecordBatchTests.swift @@ -29,8 +29,8 @@ final class RecordBatchTests: XCTestCase { stringBuilder.append("test22") stringBuilder.append("test33") - let intHolder = ArrowArrayHolder(try uint8Builder.finish()) - let stringHolder = ArrowArrayHolder(try stringBuilder.finish()) + let intHolder = ArrowArrayHolderImpl(try uint8Builder.finish()) + let stringHolder = ArrowArrayHolderImpl(try stringBuilder.finish()) let result = RecordBatch.Builder() .addColumn("col1", arrowArray: intHolder) .addColumn("col2", arrowArray: stringHolder) diff --git a/swift/Arrow/Tests/ArrowTests/TableTests.swift b/swift/Arrow/Tests/ArrowTests/TableTests.swift index a82a07979345c..8e958ccbf9f9f 100644 --- a/swift/Arrow/Tests/ArrowTests/TableTests.swift +++ b/swift/Arrow/Tests/ArrowTests/TableTests.swift @@ -132,8 +132,8 @@ final class TableTests: XCTestCase { let stringBuilder = try ArrowArrayBuilders.loadStringArrayBuilder() stringBuilder.append("test10") stringBuilder.append("test22") - let intHolder = ArrowArrayHolder(try uint8Builder.finish()) - let stringHolder = ArrowArrayHolder(try stringBuilder.finish()) + let intHolder = ArrowArrayHolderImpl(try uint8Builder.finish()) + let stringHolder = ArrowArrayHolderImpl(try stringBuilder.finish()) let result = RecordBatch.Builder() .addColumn("col1", arrowArray: intHolder) .addColumn("col2", arrowArray: stringHolder) diff --git a/swift/ArrowFlight/Package.swift b/swift/ArrowFlight/Package.swift index f3caa83486764..629b830a6e0da 100644 --- a/swift/ArrowFlight/Package.swift +++ b/swift/ArrowFlight/Package.swift @@ -29,7 +29,7 @@ let package = Package( // Products define the executables and libraries a package produces, making them visible to other packages. .library( name: "ArrowFlight", - targets: ["ArrowFlight"]), + targets: ["ArrowFlight"]) ], dependencies: [ .package(url: "https://github.com/grpc/grpc-swift.git", from: "1.15.0"), @@ -48,6 +48,6 @@ let package = Package( ]), .testTarget( name: "ArrowFlightTests", - dependencies: ["ArrowFlight"]), + dependencies: ["ArrowFlight"]) ] ) diff --git a/swift/ArrowFlight/Tests/ArrowFlightTests/FlightTest.swift b/swift/ArrowFlight/Tests/ArrowFlightTests/FlightTest.swift index 8097388c7fde1..f7bc3c1ccb0c3 100644 --- a/swift/ArrowFlight/Tests/ArrowFlightTests/FlightTest.swift +++ b/swift/ArrowFlight/Tests/ArrowFlightTests/FlightTest.swift @@ -51,9 +51,9 @@ func makeRecordBatch() throws -> RecordBatch { date32Builder.append(date2) date32Builder.append(date1) date32Builder.append(date2) - let doubleHolder = ArrowArrayHolder(try doubleBuilder.finish()) - let stringHolder = ArrowArrayHolder(try stringBuilder.finish()) - let date32Holder = ArrowArrayHolder(try date32Builder.finish()) + let doubleHolder = ArrowArrayHolderImpl(try doubleBuilder.finish()) + let stringHolder = ArrowArrayHolderImpl(try stringBuilder.finish()) + let date32Holder = ArrowArrayHolderImpl(try date32Builder.finish()) let result = RecordBatch.Builder() .addColumn("col1", arrowArray: doubleHolder) .addColumn("col2", arrowArray: stringHolder) diff --git a/swift/CDataWGo/.gitignore b/swift/CDataWGo/.gitignore new file mode 100644 index 0000000000000..0023a53406379 --- /dev/null +++ b/swift/CDataWGo/.gitignore @@ -0,0 +1,8 @@ +.DS_Store +/.build +/Packages +xcuserdata/ +DerivedData/ +.swiftpm/configuration/registries.json +.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata +.netrc diff --git a/swift/CDataWGo/Package.swift b/swift/CDataWGo/Package.swift new file mode 100644 index 0000000000000..64d29aec6b845 --- /dev/null +++ b/swift/CDataWGo/Package.swift @@ -0,0 +1,43 @@ +// swift-tools-version: 5.9 +// The swift-tools-version declares the minimum version of Swift required to build this package. + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import PackageDescription + +let package = Package( + name: "go-swift", + platforms: [ + .macOS(.v10_14) + ], + products: [ + .library( + name: "go-swift", + type: .static, + targets: ["go-swift"]) + ], + dependencies: [ + .package(path: "../Arrow") // 👈 Reference to a Local Package + ], + targets: [ + .target( + name: "go-swift", + dependencies: [ + .product(name: "Arrow", package: "Arrow") + ]) + ] +) diff --git a/swift/CDataWGo/Sources/go-swift/CDataTest.swift b/swift/CDataWGo/Sources/go-swift/CDataTest.swift new file mode 100644 index 0000000000000..b38ca7240ab60 --- /dev/null +++ b/swift/CDataWGo/Sources/go-swift/CDataTest.swift @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Arrow +import ArrowC + +@_cdecl("stringTypeFromSwift") +func stringTypeFromSwift(cSchema: UnsafePointer) { + let unsafePointer = UnsafeMutablePointer(mutating: cSchema) + let exporter = ArrowCExporter() + switch exporter.exportType(&unsafePointer.pointee, arrowType: ArrowType(ArrowType.ArrowString), name: "col1") { + case .success: + return + case .failure(let err): + fatalError("Error exporting string type from swift: \(err)") + } +} + +@_cdecl("stringTypeToSwift") +func stringTypeToSwift(cSchema: UnsafePointer) { + let importer = ArrowCImporter() + switch importer.importField(cSchema.pointee) { + case .success(let field): + if field.name != "col1" { + fatalError("Field name was incorrect expected: col1 but found: \(field.name)") + } + + if field.type.id != ArrowTypeId.string { + fatalError("Field type was incorrect expected: string but found: \(field.type.id)") + } + case .failure(let err): + fatalError("Error importing string type to swift: \(err)") + } +} + +@_cdecl("arrayIntFromSwift") +func arrayIntFromSwift(cArray: UnsafePointer) { + do { + let unsafePointer = UnsafeMutablePointer(mutating: cArray) + let arrayBuilder: NumberArrayBuilder = try ArrowArrayBuilders.loadNumberArrayBuilder() + for index in 0..<100 { + arrayBuilder.append(Int32(index)) + } + + let array = try arrayBuilder.finish() + let exporter = ArrowCExporter() + exporter.exportArray(&unsafePointer.pointee, arrowData: array.arrowData) + } catch let err { + fatalError("Error exporting array from swift \(err)") + } +} + +@_cdecl("arrayStringFromSwift") +func arrayStringFromSwift(cArray: UnsafePointer) { + do { + let unsafePointer = UnsafeMutablePointer(mutating: cArray) + let arrayBuilder = try ArrowArrayBuilders.loadStringArrayBuilder() + for index in 0..<100 { + arrayBuilder.append("test" + String(index)) + } + + let array = try arrayBuilder.finish() + let exporter = ArrowCExporter() + exporter.exportArray(&unsafePointer.pointee, arrowData: array.arrowData) + } catch let err { + fatalError("Error exporting array from swift \(err)") + } +} + +@_cdecl("arrayIntToSwift") +func arrayIntToSwift(cArray: UnsafePointer) { + let importer = ArrowCImporter() + switch importer.importArray(cArray, arrowType: ArrowType(ArrowType.ArrowInt32)) { + case .success(let int32Holder): + let result = RecordBatch.Builder() + .addColumn("col1", arrowArray: int32Holder) + .finish() + switch result { + case .success(let recordBatch): + let col1: Arrow.ArrowArray = recordBatch.data(for: 0) + for index in 0..) { + let importer = ArrowCImporter() + switch importer.importArray(cArray, arrowType: ArrowType(ArrowType.ArrowString)) { + case .success(let dataHolder): + let result = RecordBatch.Builder() + .addColumn("col1", arrowArray: dataHolder) + .finish() + switch result { + case .success(let recordBatch): + let col1: Arrow.ArrowArray = recordBatch.data(for: 0) + for index in 0.. +#include "go_swift.h" +*/ +import "C" +import ( + "strconv" + "unsafe" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/cdata" + "github.com/apache/arrow/go/v16/arrow/memory" +) + +func stringTypeFromSwift() { + arrowSchema := &cdata.CArrowSchema{} + swSchema := (*C.struct_ArrowSchema)(unsafe.Pointer(arrowSchema)) + C.stringTypeFromSwift(swSchema) + gofield, _ := cdata.ImportCArrowField(arrowSchema) + if gofield.Name != "col1" { + panic("Imported type has incorrect name") + } +} + +func stringTypeToSwift() { + arrowSchema := &cdata.CArrowSchema{} + swSchema := (*C.struct_ArrowSchema)(unsafe.Pointer(arrowSchema)) + C.stringTypeFromSwift(swSchema) + gofield, _ := cdata.ImportCArrowField(arrowSchema) + if gofield.Name != "col1" { + panic("Imported type has incorrect name") + } +} + +func arrayStringFromSwift() { + arrowArray := &cdata.CArrowArray{} + swarray := (*C.struct_ArrowArray)(unsafe.Pointer(arrowArray)) + C.arrayStringFromSwift(swarray) + arr, _ := cdata.ImportCArrayWithType(arrowArray, arrow.BinaryTypes.String) + if arr.Len() != 100 { + panic("Array length is incorrect") + } + + for i := 0; i < 100; i++ { + if arr.ValueStr(i) != ("test" + strconv.Itoa(i)) { + panic("Array value is incorrect") + } + } +} + +func arrayIntFromSwift() { + arrowArray := &cdata.CArrowArray{} + swarray := (*C.struct_ArrowArray)(unsafe.Pointer(arrowArray)) + C.arrayIntFromSwift(swarray) + arr, _ := cdata.ImportCArrayWithType(arrowArray, arrow.PrimitiveTypes.Int32) + if arr.Len() != 100 { + panic("Array length is incorrect") + } + + vals := arr.(*array.Int32).Int32Values() + // and that the values are correct + for i, v := range vals { + if v != int32(i) { + panic("Array value is incorrect") + } + } +} + +func arrayIntToSwift() { + bld := array.NewUint32Builder(memory.DefaultAllocator) + defer bld.Release() + bld.AppendValues([]uint32{1, 2, 3, 4}, []bool{true, true, true, true}) + goarray := bld.NewUint32Array() + var carray cdata.CArrowArray + cdata.ExportArrowArray(goarray, &carray, nil) + swarray := (*C.struct_ArrowArray)(unsafe.Pointer(&carray)) + C.arrayIntToSwift(swarray) + + if swarray.release != nil { + panic("Release was not called by swift to deallocate C array") + } +} + +func arrayStringToSwift() { + bld := array.NewStringBuilder(memory.DefaultAllocator) + defer bld.Release() + bld.AppendValues([]string{"test0", "test1", "test2", "test3"}, []bool{true, true, true, true}) + goarray := bld.NewStringArray() + var carray cdata.CArrowArray + cdata.ExportArrowArray(goarray, &carray, nil) + swarray := (*C.struct_ArrowArray)(unsafe.Pointer(&carray)) + C.arrayStringToSwift(swarray) + + if swarray.release != nil { + panic("Release was not called by swift to deallocate C array") + } +} + +func main() { + stringTypeFromSwift() + stringTypeToSwift() + arrayStringFromSwift() + arrayIntFromSwift() + arrayIntToSwift() + arrayStringToSwift() +} From 9f5899019d23b2b1eae2fedb9f6be8827885d843 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 29 May 2024 14:14:17 +0900 Subject: [PATCH 25/93] GH-41679: [Release][Packaging][deb] Update package name in 01-preparesh too (#41859) ### Rationale for this change It's needed when we publish minor release. For example: ```console $ dev/release/01-prepare.sh 16.0.0 17.0.0 # Release 16.0.0 ... $ dev/release/post-11-bump-versions.sh 16.0.0 17.0.0 # Released 16.0.0 ... $ dev/release/01-prepare.sh 16.1.0 17.0.0 # Release 16.1.0: This is effected ... $ dev/release/post-11-bump-versions.sh 16.1.0 17.0.0 # Released 16.1.0 ``` We can't detect minor release in `post-11-bump-versions.sh`. ### What changes are included in this PR? Share update codes via `utils-prepare.sh` and use the same logic in `01-prepare.sh` too. Linux packages related update code are also shared but it's not related to this change. Sorry. ### Are these changes tested? Yes. ### Are there any user-facing changes? Yes. * GitHub Issue: #41679 Authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- dev/release/01-prepare-test.rb | 45 ++++++++++++++-- dev/release/01-prepare.sh | 15 +++--- dev/release/post-11-bump-versions-test.rb | 8 +-- dev/release/post-11-bump-versions.sh | 50 ++---------------- dev/release/test-helper.rb | 29 +++++++++-- dev/release/utils-prepare.sh | 63 ++++++++++++++++++++++- 6 files changed, 142 insertions(+), 68 deletions(-) diff --git a/dev/release/01-prepare-test.rb b/dev/release/01-prepare-test.rb index 11e75612818ac..bf6cfede15c81 100644 --- a/dev/release/01-prepare-test.rb +++ b/dev/release/01-prepare-test.rb @@ -51,6 +51,45 @@ def prepare(*targets) sh(env, "dev/release/01-prepare.sh", @release_version, @next_version, "0") end + data(:release_type, [nil, :major, :minor, :patch]) + def test_deb_package_names + omit_on_release_branch + current_commit = git_current_commit + stdout = prepare("DEB_PACKAGE_NAMES") + changes = parse_patch(git("log", "-p", "#{current_commit}..")) + sampled_changes = changes.collect do |change| + first_hunk = change[:hunks][0] + first_removed_line = first_hunk.find { |line| line.start_with?("-") } + first_added_line = first_hunk.find { |line| line.start_with?("+") } + { + sampled_diff: [first_removed_line, first_added_line], + path: change[:path], + } + end + case release_type + when :major, :minor + expected_changes = [ + { + sampled_diff: [ + "-Package: libarrow#{@snapshot_so_version}", + "+Package: libarrow#{@so_version}", + ], + path: "dev/tasks/linux-packages/apache-arrow/debian/control.in", + }, + { + sampled_diff: [ + "- - libarrow-acero#{@snapshot_so_version}-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb", + "+ - libarrow-acero#{@so_version}-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb", + ], + path: "dev/tasks/tasks.yml", + }, + ] + else + expected_changes = [] + end + assert_equal(expected_changes, sampled_changes, "Output:\n#{stdout}") + end + def test_linux_packages user = "Arrow Developers" email = "dev@arrow.apache.org" @@ -96,7 +135,7 @@ def test_linux_packages assert_equal(expected_changes, sampled_changes, "Output:\n#{stdout}") end - data(:release_type, [:major, :minor, :patch]) + data(:next_release_type, [:major, :minor, :patch]) def test_version_pre_tag omit_on_release_branch @@ -158,7 +197,7 @@ def test_version_pre_tag ], }, ] - unless release_type == :patch + unless next_release_type == :patch expected_changes += [ { path: "docs/source/_static/versions.json", @@ -236,7 +275,7 @@ def test_version_pre_tag ], }, ] - if release_type == :major + if next_release_type == :major expected_changes += [ { path: "r/pkgdown/assets/versions.json", diff --git a/dev/release/01-prepare.sh b/dev/release/01-prepare.sh index 01fa2f3d80345..e4c62e6323c23 100755 --- a/dev/release/01-prepare.sh +++ b/dev/release/01-prepare.sh @@ -39,6 +39,7 @@ release_candidate_branch="release-${version}-rc${rc_number}" : ${PREPARE_DEFAULT:=1} : ${PREPARE_CHANGELOG:=${PREPARE_DEFAULT}} +: ${PREPARE_DEB_PACKAGE_NAMES:=${PREPARE_DEFAULT}} : ${PREPARE_LINUX_PACKAGES:=${PREPARE_DEFAULT}} : ${PREPARE_VERSION_PRE_TAG:=${PREPARE_DEFAULT}} : ${PREPARE_BRANCH:=${PREPARE_DEFAULT}} @@ -78,16 +79,12 @@ if [ ${PREPARE_CHANGELOG} -gt 0 ]; then git commit -m "MINOR: [Release] Update CHANGELOG.md for $version" fi +if [ ${PREPARE_DEB_PACKAGE_NAMES} -gt 0 ]; then + update_deb_package_names "$(current_version)" "${version}" +fi + if [ ${PREPARE_LINUX_PACKAGES} -gt 0 ]; then - echo "Updating .deb/.rpm changelogs for $version" - cd $SOURCE_DIR/../tasks/linux-packages - rake \ - version:update \ - ARROW_RELEASE_TIME="$(date +%Y-%m-%dT%H:%M:%S%z)" \ - ARROW_VERSION=${version} - git add */debian*/changelog */yum/*.spec.in - git commit -m "MINOR: [Release] Update .deb/.rpm changelogs for $version" - cd - + update_linux_packages "${version}" "$(date +%Y-%m-%dT%H:%M:%S%z)" fi if [ ${PREPARE_VERSION_PRE_TAG} -gt 0 ]; then diff --git a/dev/release/post-11-bump-versions-test.rb b/dev/release/post-11-bump-versions-test.rb index 5706b1303667a..966c723f70adf 100644 --- a/dev/release/post-11-bump-versions-test.rb +++ b/dev/release/post-11-bump-versions-test.rb @@ -74,7 +74,7 @@ def bump_versions(*targets) end end - data(:release_type, [:major, :minor, :patch]) + data(:next_release_type, [:major, :minor, :patch]) def test_version_post_tag omit_on_release_branch @@ -136,7 +136,7 @@ def test_version_post_tag ], }, ] - unless release_type == :patch + unless next_release_type == :patch expected_changes += [ { path: "docs/source/_static/versions.json", @@ -202,7 +202,7 @@ def test_version_post_tag ], }, ] - if release_type == :major + if next_release_type == :major expected_changes += [ { path: "c_glib/tool/generate-version-header.py", @@ -276,7 +276,7 @@ def test_version_post_tag import_path = "github.com/apache/arrow/go/v#{@snapshot_major_version}" hunks = [] - if release_type == :major + if next_release_type == :major lines = File.readlines(path, chomp: true) target_lines = lines.each_with_index.select do |line, i| line.include?(import_path) diff --git a/dev/release/post-11-bump-versions.sh b/dev/release/post-11-bump-versions.sh index 93eb15e0921c8..422821a66bde5 100755 --- a/dev/release/post-11-bump-versions.sh +++ b/dev/release/post-11-bump-versions.sh @@ -41,10 +41,6 @@ version=$1 next_version=$2 next_version_snapshot="${next_version}-SNAPSHOT" -current_version=$(grep ARROW_VERSION "${SOURCE_DIR}/../../cpp/CMakeLists.txt" | \ - head -n1 | \ - grep -E -o '([0-9]+\.[0-9]+\.[0-9]+)') - case "${version}" in *.0.0) is_major_release=1 @@ -68,52 +64,12 @@ if [ ${BUMP_VERSION_POST_TAG} -gt 0 ]; then fi if [ ${BUMP_DEB_PACKAGE_NAMES} -gt 0 ] && \ - [ "${next_version}" != "${current_version}" ]; then - echo "Updating .deb package names for ${next_version}" - so_version() { - local version=$1 - local major_version=$(echo $version | sed -E -e 's/^([0-9]+)\.[0-9]+\.[0-9]+$/\1/') - local minor_version=$(echo $version | sed -E -e 's/^[0-9]+\.([0-9]+)\.[0-9]+$/\1/') - expr ${major_version} \* 100 + ${minor_version} - } - deb_lib_suffix=$(so_version $version) - next_deb_lib_suffix=$(so_version $next_version) - if [ "${deb_lib_suffix}" != "${next_deb_lib_suffix}" ]; then - cd $SOURCE_DIR/../tasks/linux-packages/apache-arrow - for target in debian*/lib*${deb_lib_suffix}.install; do - git mv \ - ${target} \ - $(echo $target | sed -e "s/${deb_lib_suffix}/${next_deb_lib_suffix}/") - done - deb_lib_suffix_substitute_pattern="s/(lib(arrow|gandiva|parquet)[-a-z]*)${deb_lib_suffix}/\\1${next_deb_lib_suffix}/g" - sed -i.bak -E -e "${deb_lib_suffix_substitute_pattern}" debian*/control* - rm -f debian*/control*.bak - git add debian*/control* - cd - - cd $SOURCE_DIR/../tasks/ - sed -i.bak -E -e "${deb_lib_suffix_substitute_pattern}" tasks.yml - rm -f tasks.yml.bak - git add tasks.yml - cd - - cd $SOURCE_DIR - sed -i.bak -E -e "${deb_lib_suffix_substitute_pattern}" rat_exclude_files.txt - rm -f rat_exclude_files.txt.bak - git add rat_exclude_files.txt - git commit -m "MINOR: [Release] Update .deb package names for $next_version" - cd - - fi + [ "${next_version}" != "$(current_version)" ]; then + update_deb_package_names "${version}" "${next_version}" fi if [ ${BUMP_LINUX_PACKAGES} -gt 0 ]; then - echo "Updating .deb/.rpm changelogs for $version" - cd $SOURCE_DIR/../tasks/linux-packages - rake \ - version:update \ - ARROW_RELEASE_TIME="$(git log -n1 --format=%aI apache-arrow-${version})" \ - ARROW_VERSION=${version} - git add */debian*/changelog */yum/*.spec.in - git commit -m "MINOR: [Release] Update .deb/.rpm changelogs for $version" - cd - + update_linux_packages "${version}" "$(git log -n1 --format=%aI apache-arrow-${version})" fi if [ ${BUMP_PUSH} -gt 0 ]; then diff --git a/dev/release/test-helper.rb b/dev/release/test-helper.rb index 3b2c3aa6e5874..82400bae2793b 100644 --- a/dev/release/test-helper.rb +++ b/dev/release/test-helper.rb @@ -96,7 +96,11 @@ def parse_patch(patch) module VersionDetectable def release_type - (data || {})[:release_type] || :major + (data || {})[:release_type] + end + + def next_release_type + (data || {})[:next_release_type] || :major end def detect_versions @@ -104,19 +108,36 @@ def detect_versions cpp_cmake_lists = top_dir + "cpp" + "CMakeLists.txt" @snapshot_version = cpp_cmake_lists.read[/ARROW_VERSION "(.+?)"/, 1] @snapshot_major_version = @snapshot_version.split(".")[0] - @release_version = @snapshot_version.gsub(/-SNAPSHOT\z/, "") + @snapshot_so_version = compute_so_version(@snapshot_version.split("-")[0]) + release_version = @snapshot_version.gsub(/-SNAPSHOT\z/, "") + release_version_components = release_version.split(".") + case release_type + when nil + when :major + release_version_components[0].succ! + when :minor + release_version_components[1].succ! + when :patch + release_version_components[2].succ! + else + raise "unknown release type: #{release_type.inspect}" + end + @release_version = release_version_components.join(".") @release_compatible_version = @release_version.split(".")[0, 2].join(".") @so_version = compute_so_version(@release_version) next_version_components = @release_version.split(".") - case release_type + case next_release_type when :major next_version_components[0].succ! + next_version_components[1] = 0 + next_version_components[2] = 0 when :minor next_version_components[1].succ! + next_version_components[2] = 0 when :patch next_version_components[2].succ! else - raise "unknown release type: #{release_type.inspect}" + raise "unknown next release type: #{next_release_type.inspect}" end @next_version = next_version_components.join(".") @next_major_version = @next_version.split(".")[0] diff --git a/dev/release/utils-prepare.sh b/dev/release/utils-prepare.sh index 015f7109cd251..dfe9b052b09fa 100644 --- a/dev/release/utils-prepare.sh +++ b/dev/release/utils-prepare.sh @@ -143,7 +143,7 @@ update_versions() { DESCRIPTION rm -f DESCRIPTION.bak git add DESCRIPTION - + # Replace dev version with release version sed -i.bak -E -e \ "/^ completion counter From da0eb7e9fc90190b616aa85635d561d03e1ffe67 Mon Sep 17 00:00:00 2001 From: abandy Date: Wed, 29 May 2024 16:55:13 -0400 Subject: [PATCH 29/93] MINOR: [Swift] cleanup some go and C++ artifacts (#41878) ### Rationale for this change Follow up changes requested in the #41342 ### What changes are included in this PR? Update includes header file and go dependencies changes related to the C Data interface changes. Authored-by: Alva Bandy Signed-off-by: Sutou Kouhei --- swift/Arrow/Sources/ArrowC/ArrowCData.c | 1 + .../Arrow/Sources/ArrowC/include/ArrowCData.h | 5 +- swift/CDataWGo/go.mod | 28 ++--- swift/CDataWGo/go.sum | 106 ++++++------------ 4 files changed, 49 insertions(+), 91 deletions(-) diff --git a/swift/Arrow/Sources/ArrowC/ArrowCData.c b/swift/Arrow/Sources/ArrowC/ArrowCData.c index ac366febdaed8..fe0f80899719b 100644 --- a/swift/Arrow/Sources/ArrowC/ArrowCData.c +++ b/swift/Arrow/Sources/ArrowC/ArrowCData.c @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +#include #include "include/ArrowCData.h" void ArrowSwiftClearReleaseSchema(struct ArrowSchema* arrowSchema) { diff --git a/swift/Arrow/Sources/ArrowC/include/ArrowCData.h b/swift/Arrow/Sources/ArrowC/include/ArrowCData.h index 9df8992114be3..4b2f35efcb961 100644 --- a/swift/Arrow/Sources/ArrowC/include/ArrowCData.h +++ b/swift/Arrow/Sources/ArrowC/include/ArrowCData.h @@ -22,10 +22,7 @@ #define ARROW_FLAG_NULLABLE 2 #define ARROW_FLAG_MAP_KEYS_SORTED 4 -#include -#include -#include // Must have this! - +#include // For int64_t #ifdef __cplusplus extern "C" { diff --git a/swift/CDataWGo/go.mod b/swift/CDataWGo/go.mod index 631dd58e74bf7..323b5daac2a1e 100644 --- a/swift/CDataWGo/go.mod +++ b/swift/CDataWGo/go.mod @@ -18,24 +18,18 @@ module go-swift go 1.21 +require github.com/apache/arrow/go/v16 v16.1.0 + require ( - github.com/andybalholm/brotli v1.0.5 // indirect - github.com/apache/arrow/go/v12 v12.0.1 // indirect - github.com/apache/arrow/go/v16 v16.0.0-20240203105949-22f2cfd1e1eb // indirect - github.com/apache/thrift v0.17.0 // indirect github.com/goccy/go-json v0.10.2 // indirect - github.com/golang/snappy v0.0.4 // indirect - github.com/google/flatbuffers v23.5.26+incompatible // indirect - github.com/klauspost/asmfmt v1.3.2 // indirect - github.com/klauspost/compress v1.16.7 // indirect - github.com/klauspost/cpuid/v2 v2.2.5 // indirect - github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect - github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect - github.com/pierrec/lz4/v4 v4.1.18 // indirect + github.com/google/flatbuffers v24.3.25+incompatible // indirect + github.com/klauspost/compress v1.17.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect - golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect - golang.org/x/mod v0.13.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/tools v0.14.0 // indirect - golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect + golang.org/x/mod v0.16.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/tools v0.19.0 // indirect + golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect ) diff --git a/swift/CDataWGo/go.sum b/swift/CDataWGo/go.sum index 983c3cecaa521..e0c7e6f2747c0 100644 --- a/swift/CDataWGo/go.sum +++ b/swift/CDataWGo/go.sum @@ -1,75 +1,41 @@ -github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= -github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= -github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/apache/arrow/go/v12 v12.0.1 h1:JsR2+hzYYjgSUkBSaahpqCetqZMr76djX80fF/DiJbg= -github.com/apache/arrow/go/v12 v12.0.1/go.mod h1:weuTY7JvTG/HDPtMQxEUp7pU73vkLWMLpY67QwZ/WWw= -github.com/apache/arrow/go/v16 v16.0.0-20240202201540-0fb00fdea7a9 h1:/hG2vtror6DZi7vLBjasliWY+/3GQPnt5FrrLt+boN0= -github.com/apache/arrow/go/v16 v16.0.0-20240202201540-0fb00fdea7a9/go.mod h1:+HkSDKotr3KDBxj7gTVgj8Egy18Y1ECzQdnY5XsXwlQ= -github.com/apache/arrow/go/v16 v16.0.0-20240203105949-22f2cfd1e1eb h1:ox9Zl3OSD9yFHffSUw+mEYVnlC13je3+CGxaR5wIZmA= -github.com/apache/arrow/go/v16 v16.0.0-20240203105949-22f2cfd1e1eb/go.mod h1:+HkSDKotr3KDBxj7gTVgj8Egy18Y1ECzQdnY5XsXwlQ= -github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY= -github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= -github.com/apache/thrift v0.17.0 h1:cMd2aj52n+8VoAtvSvLn4kDC3aZ6IAkBuqWQ2IDu7wo= -github.com/apache/thrift v0.17.0/go.mod h1:OLxhMRJxomX+1I/KUw03qoV3mMz16BwaKI+d4fPBx7Q= -github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk= -github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/apache/arrow/go/v16 v16.1.0 h1:dwgfOya6s03CzH9JrjCBx6bkVb4yPD4ma3haj9p7FXI= +github.com/apache/arrow/go/v16 v16.1.0/go.mod h1:9wnc9mn6vEDTRIm4+27pEjQpRKuTvBaessPoEXQzxWA= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/flatbuffers v2.0.8+incompatible h1:ivUb1cGomAB101ZM1T0nOiWz9pSrTMoa9+EiY7igmkM= -github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= -github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= -github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= -github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= -github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= -github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= -github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= -github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= -github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= -github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= -github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= -github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= -github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI= +github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= -golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= -golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= +golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ= +gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 6800be9331d88024bf550c77865a06c592a22699 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Wed, 29 May 2024 21:30:49 -0400 Subject: [PATCH 30/93] MINOR: [R] Remove writing_bindings from _pkgdown.yml (#41877) ### Rationale for this change Missed this in #41576 ### Are these changes tested? We should make sure. ### Are there any user-facing changes? No. --- r/_pkgdown.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/r/_pkgdown.yml b/r/_pkgdown.yml index 1ce35d2a546ca..ceb68d773bdb4 100644 --- a/r/_pkgdown.yml +++ b/r/_pkgdown.yml @@ -137,7 +137,6 @@ articles: - developers/workflow - developers/debugging - developers/docker - - developers/writing_bindings - developers/install_details - developers/data_object_layout From 6c15eb8e8453b57b44c2a6975b81314978bebd8b Mon Sep 17 00:00:00 2001 From: Laurent Goujon Date: Thu, 30 May 2024 16:59:38 -0700 Subject: [PATCH 31/93] MINOR: [Java] Update develocity access key environment variable (#41880) ### Rationale for this change It was not mentioned in the migration document but `GRADLE_ENTERPRISE_ACCESS_KEY` environment variable is being deprecated and replaced with `DEVELOCITY_ACCESS_KEY` ### What changes are included in this PR? Changing github java workflows referencing the legacy variable to use the new environment variable ### Are these changes tested? No ### Are there any user-facing changes? No Authored-by: Laurent Goujon Signed-off-by: Jacob Wujciak-Jens --- .github/workflows/java.yml | 12 ++++++------ .github/workflows/java_jni.yml | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index e92d3f4fc5877..e31f7a4fc4d27 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -86,11 +86,11 @@ jobs: env: ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} run: | archery docker run \ -e CI=true \ - -e "GRADLE_ENTERPRISE_ACCESS_KEY=$GRADLE_ENTERPRISE_ACCESS_KEY" \ + -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \ ${{ matrix.image }} - name: Docker Push if: >- @@ -127,12 +127,12 @@ jobs: - name: Build shell: bash env: - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} run: ci/scripts/java_build.sh $(pwd) $(pwd)/build - name: Test shell: bash env: - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} run: ci/scripts/java_test.sh $(pwd) $(pwd)/build windows: @@ -158,10 +158,10 @@ jobs: - name: Build shell: bash env: - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} run: ci/scripts/java_build.sh $(pwd) $(pwd)/build - name: Test shell: bash env: - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} run: ci/scripts/java_test.sh $(pwd) $(pwd)/build diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml index 958216ac7669d..059a7430a38ce 100644 --- a/.github/workflows/java_jni.yml +++ b/.github/workflows/java_jni.yml @@ -120,11 +120,11 @@ jobs: env: ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} run: | archery docker run \ -e CI=true \ - -e "GRADLE_ENTERPRISE_ACCESS_KEY=$GRADLE_ENTERPRISE_ACCESS_KEY" \ + -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \ conda-python-java-integration - name: Docker Push if: >- From 706b3e09e1c7db3c7383ecb6426b86462d7abd16 Mon Sep 17 00:00:00 2001 From: Vibhatha Lakmal Abeykoon Date: Fri, 31 May 2024 07:34:46 +0530 Subject: [PATCH 32/93] GH-40932: [Java] Implement TransferPair functionality for StringView (#41861) ### Rationale for this change StringView implementation requires transferPair functionality which is required for C Data interface implementation as well. ### What changes are included in this PR? Adding transferPair functionality and corresponding test cases. ### Are these changes tested? Yes ### Are there any user-facing changes? No * GitHub Issue: #40932 Authored-by: Vibhatha Abeykoon Signed-off-by: David Li --- .../vector/BaseVariableWidthViewVector.java | 166 +++++- .../arrow/vector/ViewVarCharVector.java | 52 +- .../arrow/vector/TestSplitAndTransfer.java | 250 ++++++++- .../arrow/vector/TestVarCharViewVector.java | 498 ++++++++++++++++++ 4 files changed, 940 insertions(+), 26 deletions(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java index aaa8098b690fd..dffb4a39a9cd6 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java @@ -814,7 +814,20 @@ public TransferPair getTransferPair(BufferAllocator allocator) { * @param target destination vector for transfer */ public void transferTo(BaseVariableWidthViewVector target) { - throw new UnsupportedOperationException("trasferTo function not supported!"); + compareTypes(target, "transferTo"); + target.clear(); + target.validityBuffer = transferBuffer(validityBuffer, target.allocator); + target.viewBuffer = transferBuffer(viewBuffer, target.allocator); + target.dataBuffers = new ArrayList<>(dataBuffers.size()); + for (int i = 0; i < dataBuffers.size(); i++) { + target.dataBuffers.add(transferBuffer(dataBuffers.get(i), target.allocator)); + } + + target.setLastSet(this.lastSet); + if (this.valueCount > 0) { + target.setValueCount(this.valueCount); + } + clear(); } /** @@ -826,7 +839,154 @@ public void transferTo(BaseVariableWidthViewVector target) { */ public void splitAndTransferTo(int startIndex, int length, BaseVariableWidthViewVector target) { - throw new UnsupportedOperationException("splitAndTransferTo function not supported!"); + Preconditions.checkArgument(startIndex >= 0 && length >= 0 && startIndex + length <= valueCount, + "Invalid parameters startIndex: %s, length: %s for valueCount: %s", startIndex, length, valueCount); + compareTypes(target, "splitAndTransferTo"); + target.clear(); + if (length > 0) { + splitAndTransferValidityBuffer(startIndex, length, target); + splitAndTransferViewBufferAndDataBuffer(startIndex, length, target); + target.setLastSet(length - 1); + target.setValueCount(length); + } + } + + /* allocate validity buffer */ + private void allocateValidityBuffer(final long size) { + final int curSize = (int) size; + validityBuffer = allocator.buffer(curSize); + validityBuffer.readerIndex(0); + initValidityBuffer(); + } + + /* + * Transfer the validity. + */ + private void splitAndTransferValidityBuffer(int startIndex, int length, + BaseVariableWidthViewVector target) { + if (length <= 0) { + return; + } + + final int firstByteSource = BitVectorHelper.byteIndex(startIndex); + final int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); + final int byteSizeTarget = getValidityBufferSizeFromCount(length); + final int offset = startIndex % 8; + + if (offset == 0) { + // slice + if (target.validityBuffer != null) { + target.validityBuffer.getReferenceManager().release(); + } + final ArrowBuf slicedValidityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer = transferBuffer(slicedValidityBuffer, target.allocator); + return; + } + + /* Copy data + * When the first bit starts from the middle of a byte (offset != 0), + * copy data from src BitVector. + * Each byte in the target is composed by a part in i-th byte, + * another part in (i+1)-th byte. + */ + target.allocateValidityBuffer(byteSizeTarget); + + for (int i = 0; i < byteSizeTarget - 1; i++) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset); + + target.validityBuffer.setByte(i, (b1 + b2)); + } + /* Copying the last piece is done in the following manner: + * if the source vector has 1 or more bytes remaining, we copy + * the last piece as a byte formed by shifting data + * from the current byte and the next byte. + * + * if the source vector has no more bytes remaining + * (we are at the last byte), we copy the last piece as a byte + * by shifting data from the current byte. + */ + if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, + firstByteSource + byteSizeTarget, offset); + + target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2); + } else { + byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, + firstByteSource + byteSizeTarget - 1, offset); + target.validityBuffer.setByte(byteSizeTarget - 1, b1); + } + } + + /** + * In split and transfer, the view buffer and the data buffer will be allocated. + * Then the values will be copied from the source vector to the target vector. + * Allocation and setting are preferred over transfer + * since the buf index and buf offset needs to be overwritten + * when large strings are added. + * @param startIndex starting index + * @param length number of elements to be copied + * @param target target vector + */ + private void splitAndTransferViewBufferAndDataBuffer(int startIndex, int length, + BaseVariableWidthViewVector target) { + if (length == 0) { + return; + } + + if (target.viewBuffer != null) { + target.viewBuffer.getReferenceManager().release(); + } + + // allocate target view buffer + target.viewBuffer = target.allocator.buffer(length * ELEMENT_SIZE); + + for (int i = startIndex; i < startIndex + length; i++) { + final int stringLength = getValueLength(i); + + // keeping track of writing index in the target view buffer + int writePosition = (i - startIndex) * ELEMENT_SIZE; + // keeping track of reading index in the source view buffer + int readPosition = i * ELEMENT_SIZE; + + // set length + target.viewBuffer.setInt(writePosition, stringLength); + + if (stringLength <= INLINE_SIZE) { + // handle inline buffer + writePosition += LENGTH_WIDTH; + readPosition += LENGTH_WIDTH; + // set data by copying the required portion from the source buffer + target.viewBuffer.setBytes(writePosition, viewBuffer, readPosition, stringLength); + } else { + // handle non-inline buffer + final int readBufIndex = viewBuffer.getInt(((long) i * ELEMENT_SIZE) + + LENGTH_WIDTH + PREFIX_WIDTH); + final int readBufOffset = viewBuffer.getInt(((long) i * ELEMENT_SIZE) + + LENGTH_WIDTH + PREFIX_WIDTH + BUF_INDEX_WIDTH); + final ArrowBuf dataBuf = dataBuffers.get(readBufIndex); + + // allocate data buffer + ArrowBuf currentDataBuf = target.allocateOrGetLastDataBuffer(stringLength); + final long currentOffSet = currentDataBuf.writerIndex(); + + writePosition += LENGTH_WIDTH; + readPosition += LENGTH_WIDTH; + // set prefix + target.viewBuffer.setBytes(writePosition, viewBuffer, readPosition, PREFIX_WIDTH); + writePosition += PREFIX_WIDTH; + // set buf id + target.viewBuffer.setInt(writePosition, target.dataBuffers.size() - 1); + writePosition += BUF_INDEX_WIDTH; + // set offset + target.viewBuffer.setInt(writePosition, (int) currentOffSet); + + currentDataBuf.setBytes(currentOffSet, dataBuf, readBufOffset, stringLength); + currentDataBuf.writerIndex(currentOffSet + stringLength); + } + } } /*----------------------------------------------------------------* @@ -972,7 +1132,7 @@ public void setValueLengthSafe(int index, int length) { } /** - * Get the variable length element at specified index as Text. + * Get the length of the element at specified index. * * @param index position of an element to get * @return greater than length 0 for a non-null element, 0 otherwise diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ViewVarCharVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ViewVarCharVector.java index d35bf9e4b513b..400f8cb1fc2e0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ViewVarCharVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ViewVarCharVector.java @@ -257,9 +257,7 @@ public void validateScalars() { */ @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - // TODO: https://github.com/apache/arrow/issues/40932 - throw new UnsupportedOperationException( - "ViewVarCharVector does not support getTransferPair(String, BufferAllocator)"); + return new TransferImpl(ref, allocator); } /** @@ -271,21 +269,53 @@ public TransferPair getTransferPair(String ref, BufferAllocator allocator) { */ @Override public TransferPair getTransferPair(Field field, BufferAllocator allocator) { - // TODO: https://github.com/apache/arrow/issues/40932 - throw new UnsupportedOperationException( - "ViewVarCharVector does not support getTransferPair(Field, BufferAllocator)"); + return new TransferImpl(field, allocator); } /** * Construct a TransferPair with a desired target vector of the same type. * - * @param target the target for the transfer + * @param to the target for the transfer * @return {@link TransferPair} (UnsupportedOperationException) */ @Override - public TransferPair makeTransferPair(ValueVector target) { - // TODO: https://github.com/apache/arrow/issues/40932 - throw new UnsupportedOperationException( - "ViewVarCharVector does not support makeTransferPair(ValueVector)"); + public TransferPair makeTransferPair(ValueVector to) { + return new TransferImpl((ViewVarCharVector) to); + } + + private class TransferImpl implements TransferPair { + ViewVarCharVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new ViewVarCharVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(Field field, BufferAllocator allocator) { + to = new ViewVarCharVector(field, allocator); + } + + public TransferImpl(ViewVarCharVector to) { + this.to = to; + } + + @Override + public ViewVarCharVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, ViewVarCharVector.this); + } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java b/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java index 396f5665e0382..d2c03930ca37a 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java @@ -17,10 +17,11 @@ package org.apache.arrow.vector; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.nio.charset.StandardCharsets; import java.util.HashMap; @@ -38,20 +39,19 @@ import org.apache.arrow.vector.types.pojo.ArrowType.Struct; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.TransferPair; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestSplitAndTransfer { private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new RootAllocator(Long.MAX_VALUE); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } @@ -67,6 +67,17 @@ private void populateVarcharVector(final VarCharVector vector, int valueCount, S vector.setValueCount(valueCount); } + private void populateViewVarcharVector(final ViewVarCharVector vector, int valueCount, String[] compareArray) { + for (int i = 0; i < valueCount; i += 3) { + final String s = String.format("%010d", i); + vector.set(i, s.getBytes(StandardCharsets.UTF_8)); + if (compareArray != null) { + compareArray[i] = s; + } + } + vector.setValueCount(valueCount); + } + private void populateIntVector(final IntVector vector, int valueCount) { for (int i = 0; i < valueCount; i++) { vector.set(i, i); @@ -109,6 +120,11 @@ public void testWithEmptyVector() { transferPair = varCharVector.getTransferPair(allocator); transferPair.splitAndTransfer(0, 0); assertEquals(0, transferPair.getTo().getValueCount()); + // BaseVariableWidthViewVector + ViewVarCharVector viewVarCharVector = new ViewVarCharVector("", allocator); + transferPair = viewVarCharVector.getTransferPair(allocator); + transferPair.splitAndTransfer(0, 0); + assertEquals(0, transferPair.getTo().getValueCount()); // BaseLargeVariableWidthVector LargeVarCharVector largeVarCharVector = new LargeVarCharVector("", allocator); transferPair = largeVarCharVector.getTransferPair(allocator); @@ -209,6 +225,39 @@ public void test() throws Exception { } } + @Test + public void testView() throws Exception { + try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator)) { + viewVarCharVector.allocateNew(10000, 1000); + + final int valueCount = 500; + final String[] compareArray = new String[valueCount]; + + populateViewVarcharVector(viewVarCharVector, valueCount, compareArray); + + final TransferPair tp = viewVarCharVector.getTransferPair(allocator); + final ViewVarCharVector newViewVarCharVector = (ViewVarCharVector) tp.getTo(); + final int[][] startLengths = {{0, 201}, {201, 0}, {201, 200}, {401, 99}}; + + for (final int[] startLength : startLengths) { + final int start = startLength[0]; + final int length = startLength[1]; + tp.splitAndTransfer(start, length); + for (int i = 0; i < length; i++) { + final boolean expectedSet = ((start + i) % 3) == 0; + if (expectedSet) { + final byte[] expectedValue = compareArray[start + i].getBytes(StandardCharsets.UTF_8); + assertFalse(newViewVarCharVector.isNull(i)); + assertArrayEquals(expectedValue, newViewVarCharVector.get(i)); + } else { + assertTrue(newViewVarCharVector.isNull(i)); + } + } + newViewVarCharVector.clear(); + } + } + } + @Test public void testMemoryConstrainedTransfer() { try (final VarCharVector varCharVector = new VarCharVector("myvector", allocator)) { @@ -233,6 +282,38 @@ public void testMemoryConstrainedTransfer() { } } + @Test + public void testMemoryConstrainedTransferInViews() { + try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator)) { + // Here we have the target vector being transferred with a long string + // hence, the data buffer will be allocated. + // The default data buffer allocation takes + // BaseVariableWidthViewVector.INITIAL_VIEW_VALUE_ALLOCATION * BaseVariableWidthViewVector.ELEMENT_SIZE + // set limit = BaseVariableWidthViewVector.INITIAL_VIEW_VALUE_ALLOCATION * + // BaseVariableWidthViewVector.ELEMENT_SIZE + final int setLimit = BaseVariableWidthViewVector.INITIAL_VIEW_VALUE_ALLOCATION * + BaseVariableWidthViewVector.ELEMENT_SIZE; + allocator.setLimit(setLimit); + + viewVarCharVector.allocateNew(16000, 1000); + + final int valueCount = 1000; + + populateViewVarcharVector(viewVarCharVector, valueCount, null); + + final TransferPair tp = viewVarCharVector.getTransferPair(allocator); + final ViewVarCharVector newViewVarCharVector = (ViewVarCharVector) tp.getTo(); + final int[][] startLengths = {{0, 700}, {700, 299}}; + + for (final int[] startLength : startLengths) { + final int start = startLength[0]; + final int length = startLength[1]; + tp.splitAndTransfer(start, length); + newViewVarCharVector.clear(); + } + } + } + @Test public void testTransfer() { try (final VarCharVector varCharVector = new VarCharVector("myvector", allocator)) { @@ -264,6 +345,37 @@ public void testTransfer() { } } + @Test + public void testTransferInViews() { + try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator)) { + viewVarCharVector.allocateNew(16000, 1000); + + final int valueCount = 500; + final String[] compareArray = new String[valueCount]; + populateViewVarcharVector(viewVarCharVector, valueCount, compareArray); + + final TransferPair tp = viewVarCharVector.getTransferPair(allocator); + final ViewVarCharVector newViewVarCharVector = (ViewVarCharVector) tp.getTo(); + tp.transfer(); + + assertEquals(0, viewVarCharVector.valueCount); + assertEquals(valueCount, newViewVarCharVector.valueCount); + + for (int i = 0; i < valueCount; i++) { + final boolean expectedSet = (i % 3) == 0; + if (expectedSet) { + final byte[] expectedValue = compareArray[i].getBytes(StandardCharsets.UTF_8); + assertFalse(newViewVarCharVector.isNull(i)); + assertArrayEquals(expectedValue, newViewVarCharVector.get(i)); + } else { + assertTrue(newViewVarCharVector.isNull(i)); + } + } + + newViewVarCharVector.clear(); + } + } + @Test public void testCopyValueSafe() { try (final VarCharVector varCharVector = new VarCharVector("myvector", allocator); @@ -312,6 +424,24 @@ public void testSplitAndTransferNon() { } } + @Test + public void testSplitAndTransferNonInViews() { + try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator)) { + + viewVarCharVector.allocateNew(16000, 1000); + final int valueCount = 500; + populateViewVarcharVector(viewVarCharVector, valueCount, null); + + final TransferPair tp = viewVarCharVector.getTransferPair(allocator); + ViewVarCharVector newViewVarCharVector = (ViewVarCharVector) tp.getTo(); + + tp.splitAndTransfer(0, 0); + assertEquals(0, newViewVarCharVector.getValueCount()); + + newViewVarCharVector.clear(); + } + } + @Test public void testSplitAndTransferAll() { try (final VarCharVector varCharVector = new VarCharVector("myvector", allocator)) { @@ -330,6 +460,24 @@ public void testSplitAndTransferAll() { } } + @Test + public void testSplitAndTransferAllInViews() { + try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator)) { + + viewVarCharVector.allocateNew(16000, 1000); + final int valueCount = 500; + populateViewVarcharVector(viewVarCharVector, valueCount, null); + + final TransferPair tp = viewVarCharVector.getTransferPair(allocator); + ViewVarCharVector newViewVarCharVector = (ViewVarCharVector) tp.getTo(); + + tp.splitAndTransfer(0, valueCount); + assertEquals(valueCount, newViewVarCharVector.getValueCount()); + + newViewVarCharVector.clear(); + } + } + @Test public void testInvalidStartIndex() { try (final VarCharVector varCharVector = new VarCharVector("myvector", allocator); @@ -341,7 +489,7 @@ public void testInvalidStartIndex() { final TransferPair tp = varCharVector.makeTransferPair(newVarCharVector); - IllegalArgumentException e = Assertions.assertThrows( + IllegalArgumentException e = assertThrows( IllegalArgumentException.class, () -> tp.splitAndTransfer(valueCount, 10)); @@ -351,6 +499,27 @@ public void testInvalidStartIndex() { } } + @Test + public void testInvalidStartIndexInViews() { + try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator); + final ViewVarCharVector newViewVarCharVector = new ViewVarCharVector("newvector", allocator)) { + + viewVarCharVector.allocateNew(16000, 1000); + final int valueCount = 500; + populateViewVarcharVector(viewVarCharVector, valueCount, null); + + final TransferPair tp = viewVarCharVector.makeTransferPair(newViewVarCharVector); + + IllegalArgumentException e = assertThrows( + IllegalArgumentException.class, + () -> tp.splitAndTransfer(valueCount, 10)); + + assertEquals("Invalid parameters startIndex: 500, length: 10 for valueCount: 500", e.getMessage()); + + newViewVarCharVector.clear(); + } + } + @Test public void testInvalidLength() { try (final VarCharVector varCharVector = new VarCharVector("myvector", allocator); @@ -362,7 +531,7 @@ public void testInvalidLength() { final TransferPair tp = varCharVector.makeTransferPair(newVarCharVector); - IllegalArgumentException e = Assertions.assertThrows( + IllegalArgumentException e = assertThrows( IllegalArgumentException.class, () -> tp.splitAndTransfer(0, valueCount * 2)); @@ -372,6 +541,27 @@ public void testInvalidLength() { } } + @Test + public void testInvalidLengthInViews() { + try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator); + final ViewVarCharVector newViewVarCharVector = new ViewVarCharVector("newvector", allocator)) { + + viewVarCharVector.allocateNew(16000, 1000); + final int valueCount = 500; + populateViewVarcharVector(viewVarCharVector, valueCount, null); + + final TransferPair tp = viewVarCharVector.makeTransferPair(newViewVarCharVector); + + IllegalArgumentException e = assertThrows( + IllegalArgumentException.class, + () -> tp.splitAndTransfer(0, valueCount * 2)); + + assertEquals("Invalid parameters startIndex: 0, length: 1000 for valueCount: 500", e.getMessage()); + + newViewVarCharVector.clear(); + } + } + @Test public void testZeroStartIndexAndLength() { try (final VarCharVector varCharVector = new VarCharVector("myvector", allocator); @@ -390,6 +580,24 @@ public void testZeroStartIndexAndLength() { } } + @Test + public void testZeroStartIndexAndLengthInViews() { + try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator); + final ViewVarCharVector newViewVarCharVector = new ViewVarCharVector("newvector", allocator)) { + + viewVarCharVector.allocateNew(0, 0); + final int valueCount = 0; + populateViewVarcharVector(viewVarCharVector, valueCount, null); + + final TransferPair tp = viewVarCharVector.makeTransferPair(newViewVarCharVector); + + tp.splitAndTransfer(0, 0); + assertEquals(valueCount, newViewVarCharVector.getValueCount()); + + newViewVarCharVector.clear(); + } + } + @Test public void testZeroLength() { try (final VarCharVector varCharVector = new VarCharVector("myvector", allocator); @@ -408,6 +616,24 @@ public void testZeroLength() { } } + @Test + public void testZeroLengthInViews() { + try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator); + final ViewVarCharVector newViewVarCharVector = new ViewVarCharVector("newvector", allocator)) { + + viewVarCharVector.allocateNew(16000, 1000); + final int valueCount = 500; + populateViewVarcharVector(viewVarCharVector, valueCount, null); + + final TransferPair tp = viewVarCharVector.makeTransferPair(newViewVarCharVector); + + tp.splitAndTransfer(500, 0); + assertEquals(0, newViewVarCharVector.getValueCount()); + + newViewVarCharVector.clear(); + } + } + @Test public void testUnionVectorZeroStartIndexAndLength() { try (final UnionVector unionVector = UnionVector.empty("myvector", allocator); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharViewVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharViewVector.java index 17bc08c7d398c..1ba3bc3576fb2 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharViewVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharViewVector.java @@ -42,6 +42,7 @@ import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.memory.rounding.DefaultRoundingPolicy; import org.apache.arrow.memory.util.ArrowBufPointer; import org.apache.arrow.memory.util.CommonUtil; import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; @@ -51,6 +52,7 @@ import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.ReusableByteArray; import org.apache.arrow.vector.util.Text; +import org.apache.arrow.vector.util.TransferPair; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -1714,6 +1716,502 @@ public void testCopyFromSafeWithNulls(Function= 4096); + + /* populate the vector */ + for (int i = 0; i < valueCapacity; i++) { + if ((i & 1) == 1) { + vector.set(i, STR1); + } else { + vector.set(i, STR2); + } + } + + /* Check the vector output */ + for (int i = 0; i < valueCapacity; i++) { + if ((i & 1) == 1) { + assertArrayEquals(STR1, vector.get(i)); + } else { + assertArrayEquals(STR2, vector.get(i)); + } + } + + /* trigger first realloc */ + vector.setSafe(valueCapacity, STR2, 0, STR2.length); + assertTrue(vector.getValueCapacity() >= 2 * valueCapacity); + while (vector.getByteCapacity() < bytesPerRecord * vector.getValueCapacity()) { + vector.reallocViewBuffer(); + vector.reallocViewDataBuffer(); + } + + /* populate the remaining vector */ + for (int i = valueCapacity; i < vector.getValueCapacity(); i++) { + if ((i & 1) == 1) { + vector.set(i, STR1); + } else { + vector.set(i, STR2); + } + } + + /* Check the vector output */ + valueCapacity = vector.getValueCapacity(); + for (int i = 0; i < valueCapacity; i++) { + if ((i & 1) == 1) { + assertArrayEquals(STR1, vector.get(i)); + } else { + assertArrayEquals(STR2, vector.get(i)); + } + } + + /* trigger second realloc */ + vector.setSafe(valueCapacity + bytesPerRecord, STR2, 0, STR2.length); + assertTrue(vector.getValueCapacity() >= 2 * valueCapacity); + while (vector.getByteCapacity() < bytesPerRecord * vector.getValueCapacity()) { + vector.reallocViewBuffer(); + vector.reallocViewDataBuffer(); + } + + /* populate the remaining vector */ + for (int i = valueCapacity; i < vector.getValueCapacity(); i++) { + if ((i & 1) == 1) { + vector.set(i, STR1); + } else { + vector.set(i, STR2); + } + } + + /* Check the vector output */ + valueCapacity = vector.getValueCapacity(); + for (int i = 0; i < valueCapacity; i++) { + if ((i & 1) == 1) { + assertArrayEquals(STR1, vector.get(i)); + } else { + assertArrayEquals(STR2, vector.get(i)); + } + } + + /* We are potentially working with 4x the size of vector buffer + * that we initially started with. + * Now let's transfer the vector. + */ + + TransferPair transferPair = vector.getTransferPair(allocator); + transferPair.transfer(); + ViewVarCharVector toVector = (ViewVarCharVector) transferPair.getTo(); + valueCapacity = toVector.getValueCapacity(); + + for (int i = 0; i < valueCapacity; i++) { + if ((i & 1) == 1) { + assertArrayEquals(STR1, toVector.get(i)); + } else { + assertArrayEquals(STR2, toVector.get(i)); + } + } + + toVector.close(); + } + } + + /** + * ARROW-7831: + * ensures that data is transferred from one allocator to another in case of 0-index + * start special cases. + * With long strings and multiple data buffers. + * Check multi-data buffer source copying + */ + @Test + public void testSplitAndTransfer9() { + try (final ViewVarCharVector targetVector = new ViewVarCharVector("target", allocator)) { + String str4 = generateRandomString(35); + try (final ViewVarCharVector sourceVector = new ViewVarCharVector("source", allocator)) { + sourceVector.allocateNew(48, 4); + + sourceVector.set(0, STR1); + sourceVector.set(1, STR2); + sourceVector.set(2, STR3); + sourceVector.set(3, str4.getBytes(StandardCharsets.UTF_8)); + sourceVector.setValueCount(4); + + // we should have multiple data buffers + assertTrue(sourceVector.getDataBuffers().size() > 1); + + final long allocatedMem = allocator.getAllocatedMemory(); + final int validityRefCnt = sourceVector.getValidityBuffer().refCnt(); + final int dataRefCnt = sourceVector.getDataBuffer().refCnt(); + + // split and transfer with slice starting at the beginning: + // this should not allocate anything new + sourceVector.splitAndTransferTo(1, 3, targetVector); + // we allocate view and data buffers for the target vector + assertTrue(allocatedMem < allocator.getAllocatedMemory()); + + // the refcnts of each buffer for this test should be the same as what + // the source allocator ended up with. + assertEquals(validityRefCnt, sourceVector.getValidityBuffer().refCnt()); + // since the new view buffer is allocated, the refcnt is the same as the source vector. + assertEquals(dataRefCnt, sourceVector.getDataBuffer().refCnt()); + + assertArrayEquals(STR1, sourceVector.get(0)); + assertArrayEquals(STR2, sourceVector.get(1)); + assertArrayEquals(STR3, sourceVector.get(2)); + assertArrayEquals(str4.getBytes(StandardCharsets.UTF_8), sourceVector.get(3)); + } + assertArrayEquals(STR2, targetVector.get(0)); + assertArrayEquals(STR3, targetVector.get(1)); + assertArrayEquals(str4.getBytes(StandardCharsets.UTF_8), targetVector.get(2)); + } + } + private String generateRandomString(int length) { Random random = new Random(); StringBuilder sb = new StringBuilder(length); From 052c330de128951e2bb8202c95b5927916be66d0 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Fri, 31 May 2024 11:54:45 +0800 Subject: [PATCH 33/93] GH-41836: [Java] Fix an undefined symbol error when ARROW_S3=OFF (#41837) ### Rationale for this change Fix undefined symbol error reported at runtime after ARROW_S3=OFF is used in compiling arrow Java. ### What changes are included in this PR? ### Are these changes tested? Tested in my local. Not sure whether we need to add some test. ### Are there any user-facing changes? No. * GitHub Issue: #41836 Authored-by: PHILO-HE Signed-off-by: Sutou Kouhei --- java/dataset/src/main/cpp/jni_wrapper.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/java/dataset/src/main/cpp/jni_wrapper.cc b/java/dataset/src/main/cpp/jni_wrapper.cc index 19a43c8d2fa41..79efbeb74fc54 100644 --- a/java/dataset/src/main/cpp/jni_wrapper.cc +++ b/java/dataset/src/main/cpp/jni_wrapper.cc @@ -25,9 +25,8 @@ #include "arrow/c/helpers.h" #include "arrow/dataset/api.h" #include "arrow/dataset/file_base.h" -#include "arrow/filesystem/localfs.h" +#include "arrow/filesystem/api.h" #include "arrow/filesystem/path_util.h" -#include "arrow/filesystem/s3fs.h" #include "arrow/engine/substrait/util.h" #include "arrow/engine/substrait/serde.h" #include "arrow/engine/substrait/relation.h" @@ -660,7 +659,9 @@ JNIEXPORT void JNICALL Java_org_apache_arrow_dataset_jni_JniWrapper_releaseBuffe JNIEXPORT void JNICALL Java_org_apache_arrow_dataset_jni_JniWrapper_ensureS3Finalized( JNIEnv* env, jobject) { JNI_METHOD_START +#ifdef ARROW_S3 JniAssertOkOrThrow(arrow::fs::EnsureS3Finalized()); +#endif JNI_METHOD_END() } From 31fe24dd3345d387ba52d46c2915a909a5667813 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Fri, 31 May 2024 09:48:54 +0200 Subject: [PATCH 34/93] GH-41126: [Python] Basic bindings for Device and MemoryManager classes (#41685) ### Rationale for this change Add bindings for the C++ `arrow::Device` and `arrow::MemoryManager` classes. ### What changes are included in this PR? Basic bindings by adding the `pyarrow.Device` and `pyarrow.MemoryManager` classes, and just tested for CPU. What is not included here are additional methods on the `MemoryManager` class (eg to allocate or copy buffers), and this is also not yet tested for CUDA. Planning to do this as follow-ups, and first doing those basic bindings should enable further enhancements to be done in parallel. ### Are these changes tested? Yes, for the CPU device only. * GitHub Issue: #41126 Authored-by: Joris Van den Bossche Signed-off-by: Joris Van den Bossche --- python/pyarrow/__init__.py | 3 + python/pyarrow/device.pxi | 162 +++++++++++++++++++++++++++ python/pyarrow/includes/libarrow.pxd | 35 ++++++ python/pyarrow/io.pxi | 33 ++++++ python/pyarrow/lib.pxd | 20 ++++ python/pyarrow/lib.pyx | 3 + python/pyarrow/tests/test_device.py | 43 +++++++ python/pyarrow/tests/test_misc.py | 2 + 8 files changed, 301 insertions(+) create mode 100644 python/pyarrow/device.pxi create mode 100644 python/pyarrow/tests/test_device.py diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py index 936f4736977c8..e52e0d242bee5 100644 --- a/python/pyarrow/__init__.py +++ b/python/pyarrow/__init__.py @@ -236,6 +236,9 @@ def print_entry(label, value): RunEndEncodedScalar, ExtensionScalar) # Buffers, allocation +from pyarrow.lib import (DeviceAllocationType, Device, MemoryManager, + default_cpu_memory_manager) + from pyarrow.lib import (Buffer, ResizableBuffer, foreign_buffer, py_buffer, Codec, compress, decompress, allocate_buffer) diff --git a/python/pyarrow/device.pxi b/python/pyarrow/device.pxi new file mode 100644 index 0000000000000..6e6034752085a --- /dev/null +++ b/python/pyarrow/device.pxi @@ -0,0 +1,162 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# cython: profile=False +# distutils: language = c++ +# cython: embedsignature = True + + +cpdef enum DeviceAllocationType: + CPU = CDeviceAllocationType_kCPU + CUDA = CDeviceAllocationType_kCUDA + CUDA_HOST = CDeviceAllocationType_kCUDA_HOST + OPENCL = CDeviceAllocationType_kOPENCL + VULKAN = CDeviceAllocationType_kVULKAN + METAL = CDeviceAllocationType_kMETAL + VPI = CDeviceAllocationType_kVPI + ROCM = CDeviceAllocationType_kROCM + ROCM_HOST = CDeviceAllocationType_kROCM_HOST + EXT_DEV = CDeviceAllocationType_kEXT_DEV + CUDA_MANAGED = CDeviceAllocationType_kCUDA_MANAGED + ONEAPI = CDeviceAllocationType_kONEAPI + WEBGPU = CDeviceAllocationType_kWEBGPU + HEXAGON = CDeviceAllocationType_kHEXAGON + + +cdef object _wrap_device_allocation_type(CDeviceAllocationType device_type): + return DeviceAllocationType( device_type) + + +cdef class Device(_Weakrefable): + """ + Abstract interface for hardware devices + + This object represents a device with access to some memory spaces. + When handling a Buffer or raw memory address, it allows deciding in which + context the raw memory address should be interpreted + (e.g. CPU-accessible memory, or embedded memory on some particular GPU). + """ + + def __init__(self): + raise TypeError("Do not call Device's constructor directly, " + "use the device attribute of the MemoryManager instead.") + + cdef void init(self, const shared_ptr[CDevice]& device): + self.device = device + + @staticmethod + cdef wrap(const shared_ptr[CDevice]& device): + cdef Device self = Device.__new__(Device) + self.init(device) + return self + + def __eq__(self, other): + if not isinstance(other, Device): + return False + return self.device.get().Equals(deref((other).device.get())) + + def __repr__(self): + return "".format(frombytes(self.device.get().ToString())) + + @property + def type_name(self): + """ + A shorthand for this device's type. + """ + return frombytes(self.device.get().type_name()) + + @property + def device_id(self): + """ + A device ID to identify this device if there are multiple of this type. + + If there is no "device_id" equivalent (such as for the main CPU device on + non-numa systems) returns -1. + """ + return self.device.get().device_id() + + @property + def is_cpu(self): + """ + Whether this device is the main CPU device. + + This shorthand method is very useful when deciding whether a memory address + is CPU-accessible. + """ + return self.device.get().is_cpu() + + @property + def device_type(self): + """ + Return the DeviceAllocationType of this device. + """ + return _wrap_device_allocation_type(self.device.get().device_type()) + + +cdef class MemoryManager(_Weakrefable): + """ + An object that provides memory management primitives. + + A MemoryManager is always tied to a particular Device instance. + It can also have additional parameters (such as a MemoryPool to + allocate CPU memory). + + """ + + def __init__(self): + raise TypeError("Do not call MemoryManager's constructor directly, " + "use pyarrow.default_cpu_memory_manager() instead.") + + cdef void init(self, const shared_ptr[CMemoryManager]& mm): + self.memory_manager = mm + + @staticmethod + cdef wrap(const shared_ptr[CMemoryManager]& mm): + cdef MemoryManager self = MemoryManager.__new__(MemoryManager) + self.init(mm) + return self + + def __repr__(self): + return "".format( + frombytes(self.memory_manager.get().device().get().ToString()) + ) + + @property + def device(self): + """ + The device this MemoryManager is tied to. + """ + return Device.wrap(self.memory_manager.get().device()) + + @property + def is_cpu(self): + """ + Whether this MemoryManager is tied to the main CPU device. + + This shorthand method is very useful when deciding whether a memory + address is CPU-accessible. + """ + return self.memory_manager.get().is_cpu() + + +def default_cpu_memory_manager(): + """ + Return the default CPU MemoryManager instance. + + The returned singleton instance uses the default MemoryPool. + """ + return MemoryManager.wrap(c_default_cpu_memory_manager()) diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 8bfc31edc747d..a66f584b83f5b 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -316,6 +316,38 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: cdef cppclass CProxyMemoryPool" arrow::ProxyMemoryPool"(CMemoryPool): CProxyMemoryPool(CMemoryPool*) + ctypedef enum CDeviceAllocationType "arrow::DeviceAllocationType": + CDeviceAllocationType_kCPU "arrow::DeviceAllocationType::kCPU" + CDeviceAllocationType_kCUDA "arrow::DeviceAllocationType::kCUDA" + CDeviceAllocationType_kCUDA_HOST "arrow::DeviceAllocationType::kCUDA_HOST" + CDeviceAllocationType_kOPENCL "arrow::DeviceAllocationType::kOPENCL" + CDeviceAllocationType_kVULKAN "arrow::DeviceAllocationType::kVULKAN" + CDeviceAllocationType_kMETAL "arrow::DeviceAllocationType::kMETAL" + CDeviceAllocationType_kVPI "arrow::DeviceAllocationType::kVPI" + CDeviceAllocationType_kROCM "arrow::DeviceAllocationType::kROCM" + CDeviceAllocationType_kROCM_HOST "arrow::DeviceAllocationType::kROCM_HOST" + CDeviceAllocationType_kEXT_DEV "arrow::DeviceAllocationType::kEXT_DEV" + CDeviceAllocationType_kCUDA_MANAGED "arrow::DeviceAllocationType::kCUDA_MANAGED" + CDeviceAllocationType_kONEAPI "arrow::DeviceAllocationType::kONEAPI" + CDeviceAllocationType_kWEBGPU "arrow::DeviceAllocationType::kWEBGPU" + CDeviceAllocationType_kHEXAGON "arrow::DeviceAllocationType::kHEXAGON" + + cdef cppclass CDevice" arrow::Device": + const char* type_name() + c_string ToString() + c_bool Equals(const CDevice& other) + int64_t device_id() + c_bool is_cpu() const + shared_ptr[CMemoryManager] default_memory_manager() + CDeviceAllocationType device_type() + + cdef cppclass CMemoryManager" arrow::MemoryManager": + const shared_ptr[CDevice] device() + c_bool is_cpu() const + + shared_ptr[CMemoryManager] c_default_cpu_memory_manager \ + " arrow::default_cpu_memory_manager"() + cdef cppclass CBuffer" arrow::Buffer": CBuffer(const uint8_t* data, int64_t size) const uint8_t* data() @@ -328,6 +360,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: c_bool is_mutable() const c_string ToHexString() c_bool Equals(const CBuffer& other) + shared_ptr[CDevice] device() + const shared_ptr[CMemoryManager] memory_manager() + CDeviceAllocationType device_type() CResult[shared_ptr[CBuffer]] SliceBufferSafe( const shared_ptr[CBuffer]& buffer, int64_t offset) diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi index 9e8026deb435c..48b7934209c3a 100644 --- a/python/pyarrow/io.pxi +++ b/python/pyarrow/io.pxi @@ -1327,6 +1327,39 @@ cdef class Buffer(_Weakrefable): """ return self.buffer.get().is_cpu() + @property + def device(self): + """ + The device where the buffer resides. + + Returns + ------- + Device + """ + return Device.wrap(self.buffer.get().device()) + + @property + def memory_manager(self): + """ + The memory manager associated with the buffer. + + Returns + ------- + MemoryManager + """ + return MemoryManager.wrap(self.buffer.get().memory_manager()) + + @property + def device_type(self): + """ + The device type where the buffer resides. + + Returns + ------- + DeviceAllocationType + """ + return _wrap_device_allocation_type(self.buffer.get().device_type()) + @property def parent(self): cdef shared_ptr[CBuffer] parent_buf = self.buffer.get().parent() diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd index bfd266a807c40..1bc639cc8d2ba 100644 --- a/python/pyarrow/lib.pxd +++ b/python/pyarrow/lib.pxd @@ -524,6 +524,26 @@ cdef class RecordBatch(_Tabular): cdef void init(self, const shared_ptr[CRecordBatch]& table) +cdef class Device(_Weakrefable): + cdef: + shared_ptr[CDevice] device + + cdef void init(self, const shared_ptr[CDevice]& device) + + @staticmethod + cdef wrap(const shared_ptr[CDevice]& device) + + +cdef class MemoryManager(_Weakrefable): + cdef: + shared_ptr[CMemoryManager] memory_manager + + cdef void init(self, const shared_ptr[CMemoryManager]& memory_manager) + + @staticmethod + cdef wrap(const shared_ptr[CMemoryManager]& mm) + + cdef class Buffer(_Weakrefable): cdef: shared_ptr[CBuffer] buffer diff --git a/python/pyarrow/lib.pyx b/python/pyarrow/lib.pyx index 3245e50f0fe69..904e018ffddcc 100644 --- a/python/pyarrow/lib.pyx +++ b/python/pyarrow/lib.pyx @@ -162,6 +162,9 @@ include "pandas-shim.pxi" # Memory pools and allocation include "memory.pxi" +# Device type and memory manager +include "device.pxi" + # DataType, Field, Schema include "types.pxi" diff --git a/python/pyarrow/tests/test_device.py b/python/pyarrow/tests/test_device.py new file mode 100644 index 0000000000000..6bdb015be1a95 --- /dev/null +++ b/python/pyarrow/tests/test_device.py @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pyarrow as pa + + +def test_device_memory_manager(): + mm = pa.default_cpu_memory_manager() + assert mm.is_cpu + device = mm.device + assert device.is_cpu + assert device.device_id == -1 + assert device.device_type == pa.DeviceAllocationType.CPU + assert device.type_name == "arrow::CPUDevice" + assert device == device + assert repr(device) == "" + assert repr(mm) == "" + + +def test_buffer_device(): + arr = pa.array([0, 1, 2]) + buf = arr.buffers()[1] + assert buf.device_type == pa.DeviceAllocationType.CPU + assert isinstance(buf.device, pa.Device) + assert isinstance(buf.memory_manager, pa.MemoryManager) + assert buf.is_cpu + assert buf.device.is_cpu + assert buf.device == pa.default_cpu_memory_manager().device + assert buf.memory_manager.is_cpu diff --git a/python/pyarrow/tests/test_misc.py b/python/pyarrow/tests/test_misc.py index 39dac4eb81dfb..308c37fd0de1e 100644 --- a/python/pyarrow/tests/test_misc.py +++ b/python/pyarrow/tests/test_misc.py @@ -242,6 +242,8 @@ def test_set_timezone_db_path_non_windows(): pa.MemoryPool, pa.LoggingMemoryPool, pa.ProxyMemoryPool, + pa.Device, + pa.MemoryManager, ]) def test_extension_type_constructor_errors(klass): # ARROW-2638: prevent calling extension class constructors directly From 255dbf990c3d3e5fb1270a2a11efe0af2be195ab Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Fri, 31 May 2024 10:09:54 +0200 Subject: [PATCH 35/93] GH-41684: [C++][Python] Add optional null_bitmap to MapArray::FromArrays (#41757) ### Rationale for this change When constructing a `MapArray` with `FromArrays` one can not supply a `null_bitmap`. ### What changes are included in this PR? Optional `null_bitmap` argument is added to `MapArray::FromArrays`. ### Are these changes tested? TODO (have them locally, need to clean them up and commit. ### Are there any user-facing changes? No. * GitHub Issue: #41684 Authored-by: AlenkaF Signed-off-by: Joris Van den Bossche --- cpp/src/arrow/array/array_list_test.cc | 17 ++++++++++ cpp/src/arrow/array/array_nested.cc | 45 ++++++++++++++++++-------- cpp/src/arrow/array/array_nested.h | 9 ++++-- python/pyarrow/array.pxi | 11 +++++-- python/pyarrow/includes/libarrow.pxd | 8 +++-- python/pyarrow/tests/test_array.py | 34 +++++++++++++++++++ 6 files changed, 102 insertions(+), 22 deletions(-) diff --git a/cpp/src/arrow/array/array_list_test.cc b/cpp/src/arrow/array/array_list_test.cc index e79ce6fe172b2..55f91dc34167b 100644 --- a/cpp/src/arrow/array/array_list_test.cc +++ b/cpp/src/arrow/array/array_list_test.cc @@ -1368,6 +1368,23 @@ TEST_F(TestMapArray, FromArrays) { ASSERT_EQ(keys_with_null->length(), tmp_items->length()); ASSERT_RAISES(Invalid, MapArray::FromArrays(offsets1, keys_with_null, tmp_items, pool_)); + + // With null_bitmap + ASSERT_OK_AND_ASSIGN(auto map7, MapArray::FromArrays(offsets1, keys, items, pool_, + offsets3->data()->buffers[0])); + ASSERT_OK(map7->Validate()); + MapArray expected7(map_type, length, offsets1->data()->buffers[1], keys, items, + offsets3->data()->buffers[0], 1); + AssertArraysEqual(expected7, *map7); + + // Null bitmap and offset with null + ASSERT_RAISES(Invalid, MapArray::FromArrays(offsets3, keys, items, pool_, + offsets3->data()->buffers[0])); + + // Null bitmap and offset with offset + ASSERT_RAISES(NotImplemented, + MapArray::FromArrays(offsets3->Slice(2), keys, items, pool_, + offsets3->data()->buffers[0])); } TEST_F(TestMapArray, FromArraysEquality) { diff --git a/cpp/src/arrow/array/array_nested.cc b/cpp/src/arrow/array/array_nested.cc index 67a499c2b8277..bb5c6bf018006 100644 --- a/cpp/src/arrow/array/array_nested.cc +++ b/cpp/src/arrow/array/array_nested.cc @@ -807,7 +807,7 @@ MapArray::MapArray(const std::shared_ptr& type, int64_t length, Result> MapArray::FromArraysInternal( std::shared_ptr type, const std::shared_ptr& offsets, const std::shared_ptr& keys, const std::shared_ptr& items, - MemoryPool* pool) { + MemoryPool* pool, const std::shared_ptr& null_bitmap) { using offset_type = typename MapType::offset_type; using OffsetArrowType = typename CTypeTraits::ArrowType; @@ -827,6 +827,15 @@ Result> MapArray::FromArraysInternal( return Status::Invalid("Map key and item arrays must be equal length"); } + if (null_bitmap != nullptr && offsets->null_count() > 0) { + return Status::Invalid( + "Ambiguous to specify both validity map and offsets with nulls"); + } + + if (null_bitmap != nullptr && offsets->offset() != 0) { + return Status::NotImplemented("Null bitmap with offsets slice not supported."); + } + if (offsets->null_count() > 0) { ARROW_ASSIGN_OR_RAISE(auto buffers, CleanListOffsets(NULLPTR, *offsets, pool)); @@ -836,24 +845,32 @@ Result> MapArray::FromArraysInternal( using OffsetArrayType = typename TypeTraits::ArrayType; const auto& typed_offsets = checked_cast(*offsets); - auto buffers = BufferVector({nullptr, typed_offsets.values()}); + + BufferVector buffers; + int64_t null_count; + if (null_bitmap != nullptr) { + buffers = BufferVector({std::move(null_bitmap), typed_offsets.values()}); + null_count = null_bitmap->size(); + } else { + buffers = BufferVector({null_bitmap, typed_offsets.values()}); + null_count = 0; + } return std::make_shared(type, offsets->length() - 1, std::move(buffers), keys, - items, /*null_count=*/0, offsets->offset()); + items, /*null_count=*/null_count, offsets->offset()); } -Result> MapArray::FromArrays(const std::shared_ptr& offsets, - const std::shared_ptr& keys, - const std::shared_ptr& items, - MemoryPool* pool) { +Result> MapArray::FromArrays( + const std::shared_ptr& offsets, const std::shared_ptr& keys, + const std::shared_ptr& items, MemoryPool* pool, + const std::shared_ptr& null_bitmap) { return FromArraysInternal(std::make_shared(keys->type(), items->type()), - offsets, keys, items, pool); + offsets, keys, items, pool, null_bitmap); } -Result> MapArray::FromArrays(std::shared_ptr type, - const std::shared_ptr& offsets, - const std::shared_ptr& keys, - const std::shared_ptr& items, - MemoryPool* pool) { +Result> MapArray::FromArrays( + std::shared_ptr type, const std::shared_ptr& offsets, + const std::shared_ptr& keys, const std::shared_ptr& items, + MemoryPool* pool, const std::shared_ptr& null_bitmap) { if (type->id() != Type::MAP) { return Status::TypeError("Expected map type, got ", type->ToString()); } @@ -864,7 +881,7 @@ Result> MapArray::FromArrays(std::shared_ptr ty if (!map_type.item_type()->Equals(items->type())) { return Status::TypeError("Mismatching map items type"); } - return FromArraysInternal(std::move(type), offsets, keys, items, pool); + return FromArraysInternal(std::move(type), offsets, keys, items, pool, null_bitmap); } Status MapArray::ValidateChildData( diff --git a/cpp/src/arrow/array/array_nested.h b/cpp/src/arrow/array/array_nested.h index 5744f5fcadf05..f96b6bd3b1346 100644 --- a/cpp/src/arrow/array/array_nested.h +++ b/cpp/src/arrow/array/array_nested.h @@ -532,15 +532,18 @@ class ARROW_EXPORT MapArray : public ListArray { /// \param[in] keys Array containing key values /// \param[in] items Array containing item values /// \param[in] pool MemoryPool in case new offsets array needs to be + /// \param[in] null_bitmap Optional validity bitmap /// allocated because of null values static Result> FromArrays( const std::shared_ptr& offsets, const std::shared_ptr& keys, - const std::shared_ptr& items, MemoryPool* pool = default_memory_pool()); + const std::shared_ptr& items, MemoryPool* pool = default_memory_pool(), + const std::shared_ptr& null_bitmap = NULLPTR); static Result> FromArrays( std::shared_ptr type, const std::shared_ptr& offsets, const std::shared_ptr& keys, const std::shared_ptr& items, - MemoryPool* pool = default_memory_pool()); + MemoryPool* pool = default_memory_pool(), + const std::shared_ptr& null_bitmap = NULLPTR); const MapType* map_type() const { return map_type_; } @@ -560,7 +563,7 @@ class ARROW_EXPORT MapArray : public ListArray { static Result> FromArraysInternal( std::shared_ptr type, const std::shared_ptr& offsets, const std::shared_ptr& keys, const std::shared_ptr& items, - MemoryPool* pool); + MemoryPool* pool, const std::shared_ptr& null_bitmap = NULLPTR); private: const MapType* map_type_; diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 406830ad4dd69..3c26e85887466 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -3060,7 +3060,7 @@ cdef class MapArray(ListArray): """ @staticmethod - def from_arrays(offsets, keys, items, DataType type=None, MemoryPool pool=None): + def from_arrays(offsets, keys, items, DataType type=None, MemoryPool pool=None, mask=None): """ Construct MapArray from arrays of int32 offsets and key, item arrays. @@ -3072,6 +3072,8 @@ cdef class MapArray(ListArray): type : DataType, optional If not specified, a default MapArray with the keys' and items' type is used. pool : MemoryPool + mask : Array (boolean type), optional + Indicate which values are null (True) or not null (False). Returns ------- @@ -3153,24 +3155,27 @@ cdef class MapArray(ListArray): cdef: Array _offsets, _keys, _items shared_ptr[CArray] out + shared_ptr[CBuffer] c_mask cdef CMemoryPool* cpool = maybe_unbox_memory_pool(pool) _offsets = asarray(offsets, type='int32') _keys = asarray(keys) _items = asarray(items) + c_mask = c_mask_inverted_from_obj(mask, pool) + if type is not None: with nogil: out = GetResultValue( CMapArray.FromArraysAndType( type.sp_type, _offsets.sp_array, - _keys.sp_array, _items.sp_array, cpool)) + _keys.sp_array, _items.sp_array, cpool, c_mask)) else: with nogil: out = GetResultValue( CMapArray.FromArrays(_offsets.sp_array, _keys.sp_array, - _items.sp_array, cpool)) + _items.sp_array, cpool, c_mask)) cdef Array result = pyarrow_wrap_array(out) result.validate() return result diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index a66f584b83f5b..0d63ec6be38d8 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -823,7 +823,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: const shared_ptr[CArray]& offsets, const shared_ptr[CArray]& keys, const shared_ptr[CArray]& items, - CMemoryPool* pool) + CMemoryPool* pool, + const shared_ptr[CBuffer] null_bitmap, + ) @staticmethod CResult[shared_ptr[CArray]] FromArraysAndType" FromArrays"( @@ -831,7 +833,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: const shared_ptr[CArray]& offsets, const shared_ptr[CArray]& keys, const shared_ptr[CArray]& items, - CMemoryPool* pool) + CMemoryPool* pool, + const shared_ptr[CBuffer] null_bitmap, + ) shared_ptr[CArray] keys() shared_ptr[CArray] items() diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py index b89e0ace157af..49a00517fca9f 100644 --- a/python/pyarrow/tests/test_array.py +++ b/python/pyarrow/tests/test_array.py @@ -1079,6 +1079,40 @@ def test_map_from_arrays(): pa.int64() )) + # pass in null bitmap with type + result = pa.MapArray.from_arrays([0, 2, 2, 6], keys, items, pa.map_( + keys.type, + items.type), + mask=pa.array([False, True, False], type=pa.bool_()) + ) + assert result.equals(expected) + + # pass in null bitmap without the type + result = pa.MapArray.from_arrays([0, 2, 2, 6], keys, items, + mask=pa.array([False, True, False], + type=pa.bool_()) + ) + assert result.equals(expected) + + # error if null bitmap and offsets with nulls passed + msg1 = 'Ambiguous to specify both validity map and offsets with nulls' + with pytest.raises(pa.ArrowInvalid, match=msg1): + pa.MapArray.from_arrays(offsets, keys, items, pa.map_( + keys.type, + items.type), + mask=pa.array([False, True, False], type=pa.bool_()) + ) + + # error if null bitmap passed to sliced offset + msg2 = 'Null bitmap with offsets slice not supported.' + offsets = pa.array(offsets, pa.int32()) + with pytest.raises(pa.ArrowNotImplementedError, match=msg2): + pa.MapArray.from_arrays(offsets.slice(2), keys, items, pa.map_( + keys.type, + items.type), + mask=pa.array([False, True, False], type=pa.bool_()) + ) + # check invalid usage offsets = [0, 1, 3, 5] keys = np.arange(5) From 0d9d699250ad98cdcc5c99735e1379b1e79bc195 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Sun, 2 Jun 2024 06:51:56 +0900 Subject: [PATCH 36/93] GH-41920: [CI][JS] Add missing build directory argument (#41921) ### Rationale for this change This is a follow-up of GH-41455. ### What changes are included in this PR? Add missing build directory argument. ### Are these changes tested? Yes. ### Are there any user-facing changes? No. * GitHub Issue: #41920 Authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- .github/workflows/js.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/js.yml b/.github/workflows/js.yml index c9b7d7b742d88..e03d0c2dadce0 100644 --- a/.github/workflows/js.yml +++ b/.github/workflows/js.yml @@ -106,10 +106,10 @@ jobs: node-version: ${{ matrix.node }} - name: Build shell: bash - run: ci/scripts/js_build.sh $(pwd) + run: ci/scripts/js_build.sh $(pwd) build - name: Test shell: bash - run: ci/scripts/js_test.sh $(pwd) + run: ci/scripts/js_test.sh $(pwd) build windows: name: AMD64 Windows NodeJS ${{ matrix.node }} @@ -136,7 +136,7 @@ jobs: node-version: ${{ matrix.node }} - name: Build shell: bash - run: ci/scripts/js_build.sh $(pwd) + run: ci/scripts/js_build.sh $(pwd) build - name: Test shell: bash - run: ci/scripts/js_test.sh $(pwd) + run: ci/scripts/js_test.sh $(pwd) build From 44070eb99f0db907e1cf7a445bf7c6fea78e17fe Mon Sep 17 00:00:00 2001 From: Adam Reeve Date: Mon, 3 Jun 2024 08:53:34 +1200 Subject: [PATCH 37/93] GH-41806: [GLib][CI] Use vcpkg for C++ dependencies when building GLib libraries with MSVC (#41839) ### Rationale for this change This is more consistent as vcpkg is now used for both the Arrow C++ dependencies and GLib library dependencies, and also fixes building arrow-flight-glib and arrow-flight-sql-glib which previously failed due to a missing zlib dependency. ### What changes are included in this PR? * Configure the MSVC GLib build to use VCPKG as the dependency source for the C++ build * Configure GitHub packages based vcpkg caching (see https://learn.microsoft.com/en-us/vcpkg/users/binarycaching#quickstart-github) * Enable building Flight and Flight SQL ### Are these changes tested? NA ### Are there any user-facing changes? No Because GitHub packages is being used for caching, this will start creating GitHub packages that are visible in the main page of the Arrow GitHub repository by default. Eg. you can see them in the right panel on the crossbow repo which also runs tasks that use GitHub packages for vcpkg caching: https://github.com/ursacomputing/crossbow. The Arrow repository doesn't currently publish any packages, so it would probably be best to hide these by default to avoid confusing users and adding unnecessary noise. Someone with admin privileges on the Arrow GitHub repository would need to click the cog on the home page and hide this section: ![hide_packages](https://github.com/apache/arrow/assets/626438/69e1be77-65e7-4a17-8e9e-6d7cf90834b5) If this isn't desirable I could look into other ways of caching instead, eg. using a file based cache combined with GitHub actions cache action, or the experimental GitHub actions cache in vcpkg. * GitHub Issue: #41806 Lead-authored-by: Adam Reeve Co-authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- .github/workflows/ruby.yml | 35 +++++++++++++++++++++++++------- ci/scripts/cpp_build.sh | 1 + cpp/cmake_modules/Usevcpkg.cmake | 3 +++ 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 04f944f56c665..35c4460d47bc6 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -313,15 +313,17 @@ jobs: strategy: fail-fast: false env: + ARROW_ACERO: ON ARROW_BOOST_USE_SHARED: OFF ARROW_BUILD_BENCHMARKS: OFF ARROW_BUILD_SHARED: ON ARROW_BUILD_STATIC: OFF ARROW_BUILD_TESTS: OFF - ARROW_ACERO: ON ARROW_DATASET: ON - ARROW_FLIGHT: OFF - ARROW_FLIGHT_SQL: OFF + ARROW_DEPENDENCY_SOURCE: VCPKG + ARROW_DEPENDENCY_USE_SHARED: OFF + ARROW_FLIGHT: ON + ARROW_FLIGHT_SQL: ON ARROW_GANDIVA: OFF ARROW_HDFS: OFF ARROW_HOME: "${{ github.workspace }}/dist" @@ -337,13 +339,16 @@ jobs: ARROW_WITH_LZ4: OFF ARROW_WITH_OPENTELEMETRY: OFF ARROW_WITH_SNAPPY: ON - ARROW_WITH_ZLIB: OFF + ARROW_WITH_ZLIB: ON ARROW_WITH_ZSTD: ON - BOOST_SOURCE: BUNDLED CMAKE_CXX_STANDARD: "17" CMAKE_GENERATOR: Ninja CMAKE_INSTALL_PREFIX: "${{ github.workspace }}/dist" CMAKE_UNITY_BUILD: ON + VCPKG_BINARY_SOURCES: 'clear;nuget,GitHub,readwrite' + VCPKG_ROOT: "${{ github.workspace }}/vcpkg" + permissions: + packages: write steps: - name: Disable Crash Dialogs run: | @@ -361,7 +366,7 @@ jobs: - name: Install vcpkg shell: bash run: | - ci/scripts/install_vcpkg.sh ./vcpkg + ci/scripts/install_vcpkg.sh "${VCPKG_ROOT}" - name: Install meson run: | python -m pip install meson @@ -387,6 +392,22 @@ jobs: env: # We can invalidate the current cache by updating this. CACHE_VERSION: "2024-05-09" + - name: Setup NuGet credentials for vcpkg caching + shell: bash + run: | + $(vcpkg/vcpkg.exe fetch nuget | tail -n 1) \ + sources add \ + -source "https://nuget.pkg.github.com/$GITHUB_REPOSITORY_OWNER/index.json" \ + -storepasswordincleartext \ + -name "GitHub" \ + -username "$GITHUB_REPOSITORY_OWNER" \ + -password "${{ secrets.GITHUB_TOKEN }}" + $(vcpkg/vcpkg.exe fetch nuget | tail -n 1) \ + setapikey "${{ secrets.GITHUB_TOKEN }}" \ + -source "https://nuget.pkg.github.com/$GITHUB_REPOSITORY_OWNER/index.json" + - name: Build C++ vcpkg dependencies + run: | + vcpkg\vcpkg.exe install --triplet x64-windows --x-manifest-root cpp --x-install-root build\cpp\vcpkg_installed - name: Build C++ shell: cmd run: | @@ -396,4 +417,4 @@ jobs: shell: cmd run: | call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - bash -c "VCPKG_ROOT=\"$(pwd)/vcpkg\" ci/scripts/c_glib_build.sh $(pwd) $(pwd)/build" + bash -c "ci/scripts/c_glib_build.sh $(pwd) $(pwd)/build" diff --git a/ci/scripts/cpp_build.sh b/ci/scripts/cpp_build.sh index 6a3a53f2533cd..3ee7fbd9d19cd 100755 --- a/ci/scripts/cpp_build.sh +++ b/ci/scripts/cpp_build.sh @@ -137,6 +137,7 @@ else -DARROW_C_FLAGS_RELWITHDEBINFO="${ARROW_C_FLAGS_RELWITHDEBINFO:-}" \ -DARROW_DATASET=${ARROW_DATASET:-OFF} \ -DARROW_DEPENDENCY_SOURCE=${ARROW_DEPENDENCY_SOURCE:-AUTO} \ + -DARROW_DEPENDENCY_USE_SHARED=${ARROW_DEPENDENCY_USE_SHARED:-ON} \ -DARROW_ENABLE_THREADING=${ARROW_ENABLE_THREADING:-ON} \ -DARROW_ENABLE_TIMING_TESTS=${ARROW_ENABLE_TIMING_TESTS:-ON} \ -DARROW_EXTRA_ERROR_CONTEXT=${ARROW_EXTRA_ERROR_CONTEXT:-OFF} \ diff --git a/cpp/cmake_modules/Usevcpkg.cmake b/cpp/cmake_modules/Usevcpkg.cmake index b6192468da342..37a732f4b85a0 100644 --- a/cpp/cmake_modules/Usevcpkg.cmake +++ b/cpp/cmake_modules/Usevcpkg.cmake @@ -237,6 +237,9 @@ set(LZ4_ROOT CACHE STRING "") if(CMAKE_HOST_WIN32) + set(utf8proc_MSVC_STATIC_LIB_SUFFIX + "" + CACHE STRING "") set(LZ4_MSVC_LIB_PREFIX "" CACHE STRING "") From 02585cd11249116c00fdfe0822df4ad7da790c27 Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Mon, 3 Jun 2024 02:37:51 +0100 Subject: [PATCH 38/93] GH-39345: [C++][FS][Azure] Add support for environment credential (#41715) ### Rationale for this change Maybe be useful to support explicit environment credential (currently environment credential can be used as part of the Azure default credential flow). ### What changes are included in this PR? ### Are these changes tested? There are new unittests but no integration tests that we can actually authenticate successfully. We are relying on the Azure C++ SDK to abstracting that away. ### Are there any user-facing changes? Yes, environment credential is now available. * GitHub Issue: #39345 Authored-by: Thomas Newton Signed-off-by: Sutou Kouhei --- cpp/src/arrow/filesystem/azurefs.cc | 14 ++++++++++++++ cpp/src/arrow/filesystem/azurefs.h | 10 +++++++--- cpp/src/arrow/filesystem/azurefs_test.cc | 19 +++++++++++++++++++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/cpp/src/arrow/filesystem/azurefs.cc b/cpp/src/arrow/filesystem/azurefs.cc index 7462827d80f1e..f367bbdd3ed90 100644 --- a/cpp/src/arrow/filesystem/azurefs.cc +++ b/cpp/src/arrow/filesystem/azurefs.cc @@ -119,6 +119,8 @@ Status AzureOptions::ExtractFromUriQuery(const Uri& uri) { credential_kind = CredentialKind::kAnonymous; } else if (kv.second == "workload_identity") { credential_kind = CredentialKind::kWorkloadIdentity; + } else if (kv.second == "environment") { + credential_kind = CredentialKind::kEnvironment; } else { // Other credential kinds should be inferred from the given // parameters automatically. @@ -171,6 +173,9 @@ Status AzureOptions::ExtractFromUriQuery(const Uri& uri) { case CredentialKind::kWorkloadIdentity: RETURN_NOT_OK(ConfigureWorkloadIdentityCredential()); break; + case CredentialKind::kEnvironment: + RETURN_NOT_OK(ConfigureEnvironmentCredential()); + break; default: // Default credential break; @@ -252,6 +257,7 @@ bool AzureOptions::Equals(const AzureOptions& other) const { case CredentialKind::kClientSecret: case CredentialKind::kManagedIdentity: case CredentialKind::kWorkloadIdentity: + case CredentialKind::kEnvironment: return token_credential_->GetCredentialName() == other.token_credential_->GetCredentialName(); } @@ -337,6 +343,12 @@ Status AzureOptions::ConfigureWorkloadIdentityCredential() { return Status::OK(); } +Status AzureOptions::ConfigureEnvironmentCredential() { + credential_kind_ = CredentialKind::kEnvironment; + token_credential_ = std::make_shared(); + return Status::OK(); +} + Result> AzureOptions::MakeBlobServiceClient() const { if (account_name.empty()) { @@ -353,6 +365,7 @@ Result> AzureOptions::MakeBlobServiceC case CredentialKind::kClientSecret: case CredentialKind::kManagedIdentity: case CredentialKind::kWorkloadIdentity: + case CredentialKind::kEnvironment: return std::make_unique(AccountBlobUrl(account_name), token_credential_); case CredentialKind::kStorageSharedKey: @@ -379,6 +392,7 @@ AzureOptions::MakeDataLakeServiceClient() const { case CredentialKind::kClientSecret: case CredentialKind::kManagedIdentity: case CredentialKind::kWorkloadIdentity: + case CredentialKind::kEnvironment: return std::make_unique( AccountDfsUrl(account_name), token_credential_); case CredentialKind::kStorageSharedKey: diff --git a/cpp/src/arrow/filesystem/azurefs.h b/cpp/src/arrow/filesystem/azurefs.h index b71a5ae73b2e9..5d100bbcb4a8a 100644 --- a/cpp/src/arrow/filesystem/azurefs.h +++ b/cpp/src/arrow/filesystem/azurefs.h @@ -120,6 +120,7 @@ struct ARROW_EXPORT AzureOptions { kClientSecret, kManagedIdentity, kWorkloadIdentity, + kEnvironment, } credential_kind_ = CredentialKind::kDefault; std::shared_ptr @@ -160,11 +161,13 @@ struct ARROW_EXPORT AzureOptions { /// * dfs_storage_authority: Set AzureOptions::dfs_storage_authority /// * enable_tls: If it's "false" or "0", HTTP not HTTPS is used. /// * credential_kind: One of "default", "anonymous", - /// "workload_identity". If "default" is specified, it's just - /// ignored. If "anonymous" is specified, + /// "workload_identity" or "environment". If "default" is specified, it's + /// just ignored. If "anonymous" is specified, /// AzureOptions::ConfigureAnonymousCredential() is called. If /// "workload_identity" is specified, - /// AzureOptions::ConfigureWorkloadIdentityCredential() is called. + /// AzureOptions::ConfigureWorkloadIdentityCredential() is called, If + /// "environment" is specified, + /// AzureOptions::ConfigureEnvironmentCredential() is called. /// * tenant_id: You must specify "client_id" and "client_secret" /// too. AzureOptions::ConfigureClientSecretCredential() is called. /// * client_id: If you don't specify "tenant_id" and @@ -188,6 +191,7 @@ struct ARROW_EXPORT AzureOptions { const std::string& client_secret); Status ConfigureManagedIdentityCredential(const std::string& client_id = std::string()); Status ConfigureWorkloadIdentityCredential(); + Status ConfigureEnvironmentCredential(); bool Equals(const AzureOptions& other) const; diff --git a/cpp/src/arrow/filesystem/azurefs_test.cc b/cpp/src/arrow/filesystem/azurefs_test.cc index ed09bfc2fadd7..6075cbf0c0c91 100644 --- a/cpp/src/arrow/filesystem/azurefs_test.cc +++ b/cpp/src/arrow/filesystem/azurefs_test.cc @@ -528,6 +528,13 @@ TEST(AzureFileSystem, InitializeWithWorkloadIdentityCredential) { EXPECT_OK_AND_ASSIGN(auto fs, AzureFileSystem::Make(options)); } +TEST(AzureFileSystem, InitializeWithEnvironmentCredential) { + AzureOptions options; + options.account_name = "dummy-account-name"; + ARROW_EXPECT_OK(options.ConfigureEnvironmentCredential()); + EXPECT_OK_AND_ASSIGN(auto fs, AzureFileSystem::Make(options)); +} + TEST(AzureFileSystem, OptionsCompare) { AzureOptions options; EXPECT_TRUE(options.Equals(options)); @@ -669,6 +676,15 @@ class TestAzureOptions : public ::testing::Test { ASSERT_EQ(options.credential_kind_, AzureOptions::CredentialKind::kWorkloadIdentity); } + void TestFromUriCredentialEnvironment() { + ASSERT_OK_AND_ASSIGN( + auto options, + AzureOptions::FromUri("abfs://account.blob.core.windows.net/container/dir/blob?" + "credential_kind=environment", + nullptr)); + ASSERT_EQ(options.credential_kind_, AzureOptions::CredentialKind::kEnvironment); + } + void TestFromUriCredentialInvalid() { ASSERT_RAISES(Invalid, AzureOptions::FromUri( "abfs://file_system@account.dfs.core.windows.net/dir/file?" @@ -720,6 +736,9 @@ TEST_F(TestAzureOptions, FromUriCredentialManagedIdentity) { TEST_F(TestAzureOptions, FromUriCredentialWorkloadIdentity) { TestFromUriCredentialWorkloadIdentity(); } +TEST_F(TestAzureOptions, FromUriCredentialEnvironment) { + TestFromUriCredentialEnvironment(); +} TEST_F(TestAzureOptions, FromUriCredentialInvalid) { TestFromUriCredentialInvalid(); } TEST_F(TestAzureOptions, FromUriBlobStorageAuthority) { TestFromUriBlobStorageAuthority(); From 54bece3d4cf28df3e7bd92ed27f62d705a5cac96 Mon Sep 17 00:00:00 2001 From: Vibhatha Lakmal Abeykoon Date: Mon, 3 Jun 2024 16:55:54 +0530 Subject: [PATCH 39/93] GH-41648: [Java] Memory Leak about splitAndTransfer (#41898) ### Rationale for this change An inconsistency in the validity buffer split and transfer functionality in the `BaseFixedWidthVector`. ### What changes are included in this PR? - [X] Change to handle the validity buffer transfer - [X] User provided test criterion which reproduced the error - [X] Upgrading to JUnit 5 ### Are these changes tested? Mainly by existing test cases and added new one. ### Are there any user-facing changes? No * GitHub Issue: #41648 Authored-by: Vibhatha Abeykoon Signed-off-by: David Li --- .../arrow/vector/BaseFixedWidthVector.java | 4 +- .../apache/arrow/vector/TestValueVector.java | 197 +++++++++++------- 2 files changed, 122 insertions(+), 79 deletions(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java index c456c625389ba..a6e1a71dc36bd 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java @@ -674,8 +674,8 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, if (target.validityBuffer != null) { target.validityBuffer.getReferenceManager().release(); } - target.validityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); - target.validityBuffer.getReferenceManager().retain(1); + ArrowBuf slicedValidityBuffer = validityBuffer.slice(firstByteSource, byteSizeTarget); + target.validityBuffer = transferBuffer(slicedValidityBuffer, target.allocator); target.refreshValueCapacity(); } else { /* Copy data diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index 3e53512f7338f..fda14b24a4c8b 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -21,12 +21,13 @@ import static org.apache.arrow.vector.TestUtils.newVarCharVector; import static org.apache.arrow.vector.TestUtils.newVector; import static org.apache.arrow.vector.testing.ValueVectorDataPopulator.setVector; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.nio.ByteBuffer; import java.nio.charset.Charset; @@ -69,9 +70,9 @@ import org.apache.arrow.vector.util.ReusableByteArray; import org.apache.arrow.vector.util.Text; import org.apache.arrow.vector.util.TransferPair; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestValueVector { @@ -79,7 +80,7 @@ public class TestValueVector { private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new RootAllocator(Long.MAX_VALUE); } @@ -95,7 +96,7 @@ public void init() { (int) (Integer.getInteger("arrow.vector.max_allocation_bytes", Integer.MAX_VALUE) / 7); private static final int MAX_VALUE_COUNT_8BYTE = (int) (MAX_VALUE_COUNT / 2); - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } @@ -188,7 +189,7 @@ public void testFixedType1() { /* vector data should have been zeroed out */ for (int i = 0; i < capacityBeforeReset; i++) { // TODO: test vector.get(i) is 0 after unsafe get added - assertEquals("non-zero data not expected at index: " + i, true, vector.isNull(i)); + assertTrue(vector.isNull(i), "non-zero data not expected at index: " + i); } } } @@ -276,7 +277,7 @@ public void testFixedType2() { /* check vector contents */ j = 1; for (int i = 0; i < initialCapacity; i += 2) { - assertEquals("unexpected value at index: " + i, j, intVector.get(i)); + assertEquals(j, intVector.get(i), "unexpected value at index: " + i); j++; } @@ -298,7 +299,7 @@ public void testFixedType2() { /* vector data should still be intact after realloc */ j = 1; for (int i = 0; i <= initialCapacity; i += 2) { - assertEquals("unexpected value at index: " + i, j, intVector.get(i)); + assertEquals(j, intVector.get(i), "unexpected value at index: " + i); j++; } @@ -311,7 +312,7 @@ public void testFixedType2() { /* vector data should have been zeroed out */ for (int i = 0; i < capacityBeforeRealloc; i++) { - assertEquals("non-zero data not expected at index: " + i, true, intVector.isNull(i)); + assertTrue(intVector.isNull(i), "non-zero data not expected at index: " + i); } } } @@ -427,7 +428,7 @@ public void testFixedFloat2() { /* vector data should be zeroed out */ for (int i = 0; i < capacityBeforeReset; i++) { - assertTrue("non-zero data not expected at index: " + i, floatVector.isNull(i)); + assertTrue(floatVector.isNull(i), "non-zero data not expected at index: " + i); } } } @@ -526,7 +527,7 @@ public void testFixedFloat2WithPossibleTruncate() { /* vector data should be zeroed out */ for (int i = 0; i < capacityBeforeReset; i++) { - assertTrue("non-zero data not expected at index: " + i, floatVector.isNull(i)); + assertTrue(floatVector.isNull(i), "non-zero data not expected at index: " + i); } } } @@ -626,7 +627,7 @@ public void testFixedType3() { /* vector data should be zeroed out */ for (int i = 0; i < capacityBeforeReset; i++) { - assertEquals("non-zero data not expected at index: " + i, true, floatVector.isNull(i)); + assertTrue(floatVector.isNull(i), "non-zero data not expected at index: " + i); } } } @@ -724,7 +725,7 @@ public void testFixedType4() { /* vector data should be zeroed out */ for (int i = 0; i < capacityBeforeReset; i++) { - assertEquals("non-zero data not expected at index: " + i, true, floatVector.isNull(i)); + assertTrue(floatVector.isNull(i), "non-zero data not expected at index: " + i); } } } @@ -821,7 +822,7 @@ public void testNullableFixedType1() { /* vector data should be zeroed out */ for (int i = 0; i < capacityBeforeReset; i++) { - assertTrue("non-null data not expected at index: " + i, vector.isNull(i)); + assertTrue(vector.isNull(i), "non-null data not expected at index: " + i); } } } @@ -920,7 +921,7 @@ public void testNullableFixedType2() { /* vector data should be zeroed out */ for (int i = 0; i < capacityBeforeReset; i++) { - assertTrue("non-null data not expected at index: " + i, vector.isNull(i)); + assertTrue(vector.isNull(i), "non-null data not expected at index: " + i); } } } @@ -949,10 +950,10 @@ public void testNullableFixedType3() { int j = 1; for (int i = 0; i <= 1023; i++) { if ((i >= 2 && i <= 99) || (i >= 101 && i <= 1021)) { - assertTrue("non-null data not expected at index: " + i, vector.isNull(i)); + assertTrue(vector.isNull(i), "non-null data not expected at index: " + i); } else { - assertFalse("null data not expected at index: " + i, vector.isNull(i)); - assertEquals("unexpected value at index: " + i, j, vector.get(i)); + assertFalse(vector.isNull(i), "null data not expected at index: " + i); + assertEquals(j, vector.get(i), "unexpected value at index: " + i); j++; } } @@ -987,10 +988,10 @@ public void testNullableFixedType3() { j = 1; for (int i = 0; i < (initialCapacity * 2); i++) { if ((i > 1023 && i != initialCapacity) || (i >= 2 && i <= 99) || (i >= 101 && i <= 1021)) { - assertTrue("non-null data not expected at index: " + i, vector.isNull(i)); + assertTrue(vector.isNull(i), "non-null data not expected at index: " + i); } else { - assertFalse("null data not expected at index: " + i, vector.isNull(i)); - assertEquals("unexpected value at index: " + i, j, vector.get(i)); + assertFalse(vector.isNull(i), "null data not expected at index: " + i); + assertEquals(j, vector.get(i), "unexpected value at index: " + i); j++; } } @@ -1004,13 +1005,13 @@ public void testNullableFixedType3() { /* vector data should have been zeroed out */ for (int i = 0; i < capacityBeforeReset; i++) { - assertTrue("non-null data not expected at index: " + i, vector.isNull(i)); + assertTrue(vector.isNull(i), "non-null data not expected at index: " + i); } vector.allocateNew(initialCapacity * 4); // vector has been erased for (int i = 0; i < initialCapacity * 4; i++) { - assertTrue("non-null data not expected at index: " + i, vector.isNull(i)); + assertTrue(vector.isNull(i), "non-null data not expected at index: " + i); } } } @@ -1036,10 +1037,10 @@ public void testNullableFixedType4() { for (int i = 0; i < valueCapacity; i++) { if ((i & 1) == 1) { - assertFalse("unexpected null value at index: " + i, vector.isNull(i)); - assertEquals("unexpected value at index: " + i, (baseValue + i), vector.get(i)); + assertFalse(vector.isNull(i), "unexpected null value at index: " + i); + assertEquals((baseValue + i), vector.get(i), "unexpected value at index: " + i); } else { - assertTrue("unexpected non-null value at index: " + i, vector.isNull(i)); + assertTrue(vector.isNull(i), "unexpected non-null value at index: " + i); } } @@ -1048,15 +1049,15 @@ public void testNullableFixedType4() { for (int i = 0; i < vector.getValueCapacity(); i++) { if (i == valueCapacity) { - assertFalse("unexpected null value at index: " + i, vector.isNull(i)); - assertEquals("unexpected value at index: " + i, 20000000, vector.get(i)); + assertFalse(vector.isNull(i), "unexpected null value at index: " + i); + assertEquals(20000000, vector.get(i), "unexpected value at index: " + i); } else if (i < valueCapacity) { if ((i & 1) == 1) { - assertFalse("unexpected null value at index: " + i, vector.isNull(i)); - assertEquals("unexpected value at index: " + i, (baseValue + i), vector.get(i)); + assertFalse(vector.isNull(i), "unexpected null value at index: " + i); + assertEquals((baseValue + i), vector.get(i), "unexpected value at index: " + i); } } else { - assertTrue("unexpected non-null value at index: " + i, vector.isNull(i)); + assertTrue(vector.isNull(i), "unexpected non-null value at index: " + i); } } @@ -1068,10 +1069,10 @@ public void testNullableFixedType4() { for (int i = 0; i < vector.getValueCapacity(); i++) { if (i % 2 == 0) { - assertFalse("unexpected null value at index: " + i, vector.isNull(i)); - assertEquals("unexpected value at index: " + i, (baseValue + i), vector.get(i)); + assertFalse(vector.isNull(i), "unexpected null value at index: " + i); + assertEquals((baseValue + i), vector.get(i), "unexpected value at index: " + i); } else { - assertTrue("unexpected non-null value at index: " + i, vector.isNull(i)); + assertTrue(vector.isNull(i), "unexpected non-null value at index: " + i); } } @@ -1081,13 +1082,13 @@ public void testNullableFixedType4() { for (int i = 0; i < vector.getValueCapacity(); i++) { if (i == (valueCapacityBeforeRealloc + 1000)) { - assertFalse("unexpected null value at index: " + i, vector.isNull(i)); - assertEquals("unexpected value at index: " + i, 400000000, vector.get(i)); + assertFalse(vector.isNull(i), "unexpected null value at index: " + i); + assertEquals(400000000, vector.get(i), "unexpected value at index: " + i); } else if (i < valueCapacityBeforeRealloc && (i % 2) == 0) { - assertFalse("unexpected null value at index: " + i, vector.isNull(i)); - assertEquals("unexpected value at index: " + i, baseValue + i, vector.get(i)); + assertFalse(vector.isNull(i), "unexpected null value at index: " + i); + assertEquals(baseValue + i, vector.get(i), "unexpected value at index: " + i); } else { - assertTrue("unexpected non-null value at index: " + i, vector.isNull(i)); + assertTrue(vector.isNull(i), "unexpected non-null value at index: " + i); } } @@ -1100,7 +1101,7 @@ public void testNullableFixedType4() { /* vector data should be zeroed out */ for (int i = 0; i < valueCapacityBeforeReset; i++) { - assertTrue("non-null data not expected at index: " + i, vector.isNull(i)); + assertTrue(vector.isNull(i), "non-null data not expected at index: " + i); } } } @@ -1347,23 +1348,24 @@ public void testNullableVarType2() { } } - @Test(expected = OversizedAllocationException.class) + @Test public void testReallocateCheckSuccess() { + assertThrows(OversizedAllocationException.class, () -> { + // Create a new value vector for 1024 integers. + try (final VarBinaryVector vector = newVarBinaryVector(EMPTY_SCHEMA_PATH, allocator)) { + vector.allocateNew(1024 * 10, 1024); - // Create a new value vector for 1024 integers. - try (final VarBinaryVector vector = newVarBinaryVector(EMPTY_SCHEMA_PATH, allocator)) { - vector.allocateNew(1024 * 10, 1024); + vector.set(0, STR1); + // Check the sample strings. + assertArrayEquals(STR1, vector.get(0)); - vector.set(0, STR1); - // Check the sample strings. - assertArrayEquals(STR1, vector.get(0)); + // update the index offset to a larger one + ArrowBuf offsetBuf = vector.getOffsetBuffer(); + offsetBuf.setInt(VarBinaryVector.OFFSET_WIDTH, Integer.MAX_VALUE - 5); - // update the index offset to a larger one - ArrowBuf offsetBuf = vector.getOffsetBuffer(); - offsetBuf.setInt(VarBinaryVector.OFFSET_WIDTH, Integer.MAX_VALUE - 5); - - vector.setValueLengthSafe(1, 6); - } + vector.setValueLengthSafe(1, 6); + } + }); } @Test @@ -1551,9 +1553,9 @@ public void testReallocAfterVectorTransfer2() { /* check toVector contents before realloc */ for (int i = 0; i < toVector.getValueCapacity(); i++) { - assertFalse("unexpected null value at index: " + i, toVector.isNull(i)); + assertFalse(toVector.isNull(i), "unexpected null value at index: " + i); double value = toVector.get(i); - assertEquals("unexpected value at index: " + i, baseValue + (double) i, value, 0); + assertEquals(baseValue + (double) i, value, 0, "unexpected value at index: " + i); } /* now let's realloc the toVector and check contents again */ @@ -1562,11 +1564,11 @@ public void testReallocAfterVectorTransfer2() { for (int i = 0; i < toVector.getValueCapacity(); i++) { if (i < capacityAfterRealloc2) { - assertFalse("unexpected null value at index: " + i, toVector.isNull(i)); + assertFalse(toVector.isNull(i), "unexpected null value at index: " + i); double value = toVector.get(i); - assertEquals("unexpected value at index: " + i, baseValue + (double) i, value, 0); + assertEquals(baseValue + (double) i, value, 0, "unexpected value at index: " + i); } else { - assertTrue("unexpected non-null value at index: " + i, toVector.isNull(i)); + assertTrue(toVector.isNull(i), "unexpected non-null value at index: " + i); } } @@ -1921,7 +1923,7 @@ public void testCopyFromWithNulls() { if (i % 3 == 0) { assertNull(vector.getObject(i)); } else { - assertEquals("unexpected value at index: " + i, Integer.toString(i), vector.getObject(i).toString()); + assertEquals(Integer.toString(i), vector.getObject(i).toString(), "unexpected value at index: " + i); } } @@ -1935,7 +1937,7 @@ public void testCopyFromWithNulls() { if (i % 3 == 0) { assertNull(vector2.getObject(i)); } else { - assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString()); + assertEquals(Integer.toString(i), vector2.getObject(i).toString(), "unexpected value at index: " + i); } } @@ -1948,7 +1950,7 @@ public void testCopyFromWithNulls() { if (i % 3 == 0) { assertNull(vector2.getObject(i)); } else { - assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString()); + assertEquals(Integer.toString(i), vector2.getObject(i).toString(), "unexpected value at index: " + i); } } } @@ -1982,7 +1984,7 @@ public void testCopyFromWithNulls1() { if (i % 3 == 0) { assertNull(vector.getObject(i)); } else { - assertEquals("unexpected value at index: " + i, Integer.toString(i), vector.getObject(i).toString()); + assertEquals(Integer.toString(i), vector.getObject(i).toString(), "unexpected value at index: " + i); } } @@ -2000,7 +2002,7 @@ public void testCopyFromWithNulls1() { if (i % 3 == 0) { assertNull(vector2.getObject(i)); } else { - assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString()); + assertEquals(Integer.toString(i), vector2.getObject(i).toString(), "unexpected value at index: " + i); } } @@ -2013,7 +2015,7 @@ public void testCopyFromWithNulls1() { if (i % 3 == 0) { assertNull(vector2.getObject(i)); } else { - assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString()); + assertEquals(Integer.toString(i), vector2.getObject(i).toString(), "unexpected value at index: " + i); } } } @@ -3021,16 +3023,18 @@ public void testUnionVectorEquals() { } } - @Test(expected = IllegalArgumentException.class) + @Test public void testEqualsWithIndexOutOfRange() { - try (final IntVector vector1 = new IntVector("int", allocator); - final IntVector vector2 = new IntVector("int", allocator)) { + assertThrows(IllegalArgumentException.class, () -> { + try (final IntVector vector1 = new IntVector("int", allocator); + final IntVector vector2 = new IntVector("int", allocator)) { - setVector(vector1, 1, 2); - setVector(vector2, 1, 2); + setVector(vector1, 1, 2); + setVector(vector2, 1, 2); - assertTrue(new RangeEqualsVisitor(vector1, vector2).rangeEquals(new Range(2, 3, 1))); - } + assertTrue(new RangeEqualsVisitor(vector1, vector2).rangeEquals(new Range(2, 3, 1))); + } + }); } @Test @@ -3398,4 +3402,43 @@ public void testSetGetUInt4() { assertEquals(expected, vector.getValueAsLong(1)); } } + + @Test + public void testSplitAndTransferFixedWithVector1() { + RootAllocator allocator = new RootAllocator(Long.MAX_VALUE); + try (BufferAllocator child = allocator.newChildAllocator("child", 0, Long.MAX_VALUE)) { + try (IntVector vector = new IntVector("vector", child)) { + vector.setSafe(0, 1); + vector.setSafe(1, 2); + vector.setSafe(2, 3); + vector.setValueCount(3); + + TransferPair transferPair = vector.getTransferPair(allocator); + transferPair.splitAndTransfer(0, 1); + try (IntVector target = (IntVector) transferPair.getTo()) { + // no-op try-with-resource + assertEquals(1, target.get(0)); + } + } + } + } + + @Test + public void testSplitAndTransferFixedWithVector2() { + IntVector target; + try (BufferAllocator child = allocator.newChildAllocator("child", 0, Long.MAX_VALUE)) { + try (IntVector vector = new IntVector("source", child)) { + vector.setSafe(0, 1); + vector.setSafe(1, 2); + vector.setSafe(2, 3); + vector.setValueCount(3); + + TransferPair transferPair = vector.getTransferPair(allocator); + transferPair.splitAndTransfer(0, 1); + target = (IntVector) transferPair.getTo(); + assertEquals(1, target.get(0)); + } + } + target.close(); + } } From 99014abd193c84aee7490a36fd7914389b84cfcd Mon Sep 17 00:00:00 2001 From: Tom Scott-Coombes <62209801+tscottcoombes1@users.noreply.github.com> Date: Mon, 3 Jun 2024 16:36:29 +0100 Subject: [PATCH 40/93] GH-41887: [Go] Run linter via pre-commit (#41888) ### Rationale for this change Add and run pre-commit ### What changes are included in this PR? Update pre-commit file Run pre-commit ### Are these changes tested? N/A ### Are there any user-facing changes? No GitHub Issue: https://github.com/apache/arrow/issues/41887 * GitHub Issue: #41887 Lead-authored-by: Tom Scott-Coombes Co-authored-by: Tom Scott-Coombes <62209801+tscottcoombes1@users.noreply.github.com> Co-authored-by: Sutou Kouhei Signed-off-by: Matt Topol --- .gitignore | 5 + .golangci.yaml | 29 + .pre-commit-config.yaml | 14 + go/arrow/array/bufferbuilder_numeric_test.go | 2 +- go/arrow/array/numeric.gen.go | 2 +- go/arrow/array/numeric_test.go | 12 +- go/arrow/array/numericbuilder.gen_test.go | 8 +- go/arrow/array/record_test.go | 2 +- go/arrow/datatype_nested_test.go | 16 +- go/arrow/flight/gen/flight/Flight.pb.go | 5 +- go/arrow/flight/gen/flight/FlightSql.pb.go | 5 +- go/arrow/flight/gen/flight/Flight_grpc.pb.go | 1 + go/arrow/float16/float16.go | 2 +- go/arrow/gen-flatbuffers.go | 1 + go/arrow/internal/debug/assert_off.go | 1 + go/arrow/internal/debug/assert_on.go | 1 + go/arrow/internal/debug/doc.go | 6 +- go/arrow/internal/debug/log_off.go | 1 + go/arrow/internal/debug/log_on.go | 1 + go/arrow/internal/debug/util.go | 1 + go/arrow/internal/flatbuf/Binary.go | 2 +- go/arrow/internal/flatbuf/BinaryView.go | 14 +- go/arrow/internal/flatbuf/Block.go | 19 +- go/arrow/internal/flatbuf/BodyCompression.go | 18 +- .../internal/flatbuf/BodyCompressionMethod.go | 6 +- go/arrow/internal/flatbuf/Buffer.go | 34 +- go/arrow/internal/flatbuf/Date.go | 12 +- go/arrow/internal/flatbuf/Decimal.go | 24 +- go/arrow/internal/flatbuf/DictionaryBatch.go | 24 +- .../internal/flatbuf/DictionaryEncoding.go | 48 +- go/arrow/internal/flatbuf/DictionaryKind.go | 10 +- go/arrow/internal/flatbuf/Endianness.go | 4 +- go/arrow/internal/flatbuf/Feature.go | 38 +- go/arrow/internal/flatbuf/Field.go | 34 +- go/arrow/internal/flatbuf/FieldNode.go | 40 +- go/arrow/internal/flatbuf/FixedSizeBinary.go | 4 +- go/arrow/internal/flatbuf/FixedSizeList.go | 4 +- go/arrow/internal/flatbuf/Footer.go | 10 +- go/arrow/internal/flatbuf/KeyValue.go | 6 +- go/arrow/internal/flatbuf/LargeBinary.go | 4 +- go/arrow/internal/flatbuf/LargeList.go | 4 +- go/arrow/internal/flatbuf/LargeListView.go | 4 +- go/arrow/internal/flatbuf/LargeUtf8.go | 4 +- go/arrow/internal/flatbuf/ListView.go | 6 +- go/arrow/internal/flatbuf/Map.go | 54 +- go/arrow/internal/flatbuf/MessageHeader.go | 16 +- go/arrow/internal/flatbuf/Null.go | 2 +- go/arrow/internal/flatbuf/RecordBatch.go | 102 +- go/arrow/internal/flatbuf/RunEndEncoded.go | 10 +- go/arrow/internal/flatbuf/Schema.go | 20 +- .../internal/flatbuf/SparseMatrixIndexCSR.go | 134 +- .../internal/flatbuf/SparseMatrixIndexCSX.go | 142 +- go/arrow/internal/flatbuf/SparseTensor.go | 28 +- .../internal/flatbuf/SparseTensorIndexCOO.go | 100 +- .../internal/flatbuf/SparseTensorIndexCSF.go | 254 +- go/arrow/internal/flatbuf/Struct_.go | 6 +- go/arrow/internal/flatbuf/Tensor.go | 24 +- go/arrow/internal/flatbuf/TensorDim.go | 14 +- go/arrow/internal/flatbuf/Time.go | 28 +- go/arrow/internal/flatbuf/Timestamp.go | 250 +- go/arrow/internal/flatbuf/Type.go | 6 +- go/arrow/internal/flatbuf/Union.go | 8 +- go/arrow/internal/flatbuf/Utf8.go | 2 +- go/arrow/internal/flatbuf/Utf8View.go | 14 +- .../internal/flight_integration/scenario.go | 2 +- go/arrow/ipc/cmd/arrow-cat/main.go | 66 +- go/arrow/ipc/cmd/arrow-ls/main.go | 62 +- go/arrow/math/math_amd64.go | 1 + go/arrow/math/math_arm64.go | 5 +- go/arrow/math/math_noasm.go | 1 + go/arrow/math/math_ppc64le.go | 1 + go/arrow/math/math_s390x.go | 1 + go/arrow/memory/cgo_allocator.go | 4 +- go/arrow/memory/cgo_allocator_defaults.go | 5 +- go/arrow/memory/cgo_allocator_logging.go | 5 +- go/arrow/memory/cgo_allocator_test.go | 4 +- go/arrow/memory/memory_amd64.go | 1 + go/arrow/memory/memory_arm64.go | 1 + go/arrow/memory/memory_avx2_amd64.go | 1 + go/arrow/memory/memory_js_wasm.go | 1 + go/arrow/memory/memory_neon_arm64.go | 1 + go/arrow/memory/memory_noasm.go | 1 + go/arrow/memory/memory_sse4_amd64.go | 1 + go/internal/utils/min_max_arm64.go | 3 +- go/parquet/doc.go | 15 +- go/parquet/internal/bmi/bitmap_bmi2_amd64.go | 1 + go/parquet/internal/bmi/bitmap_bmi2_noasm.go | 1 + .../internal/bmi/bitmap_bmi2_ppc64le.go | 1 + go/parquet/internal/bmi/bitmap_bmi2_s390x.go | 1 + go/parquet/internal/bmi/bmi_amd64.go | 1 + go/parquet/internal/debug/assert_off.go | 1 + go/parquet/internal/debug/assert_on.go | 1 + go/parquet/internal/debug/doc.go | 2 +- go/parquet/internal/debug/log_off.go | 1 + go/parquet/internal/debug/log_on.go | 1 + .../encoding/delta_byte_array_test.go | 3 +- .../gen-go/parquet/GoUnusedProtection__.go | 3 +- .../internal/gen-go/parquet/parquet-consts.go | 8 +- go/parquet/internal/gen-go/parquet/parquet.go | 18925 +++++++++------- .../internal/utils/bit_packing_avx2_amd64.go | 1 + .../internal/utils/bit_packing_neon_arm64.go | 1 + .../internal/utils/unpack_bool_amd64.go | 1 + .../internal/utils/unpack_bool_arm64.go | 6 +- .../internal/utils/unpack_bool_avx2_amd64.go | 1 + .../internal/utils/unpack_bool_neon_arm64.go | 1 + .../internal/utils/unpack_bool_noasm.go | 1 + .../internal/utils/unpack_bool_sse4_amd64.go | 1 + go/parquet/metadata/app_version.go | 3 +- go/parquet/schema/reflection.go | 4 +- go/parquet/tools.go | 1 + swift/data-generator/swift-datagen/main.go | 13 +- 111 files changed, 11252 insertions(+), 9616 deletions(-) create mode 100644 .golangci.yaml diff --git a/.gitignore b/.gitignore index a482f5503c2b9..3192069d1ac7a 100644 --- a/.gitignore +++ b/.gitignore @@ -102,4 +102,9 @@ __debug_bin .envrc # Develocity +.mvn/.gradle-enterprise/ .mvn/.develocity/ + +# rat +filtered_rat.txt +rat.txt diff --git a/.golangci.yaml b/.golangci.yaml new file mode 100644 index 0000000000000..7d486a9e85a0a --- /dev/null +++ b/.golangci.yaml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +linters: + # Disable all linters. + # Default: false + disable-all: true + # Enable specific linter + # https://golangci-lint.run/usage/linters/#enabled-by-default + enable: + - gofmt + - goimports + +issues: + fix: true \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e0b8009b03184..05bf8e54f9cdb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -141,3 +141,17 @@ repos: '--disable', 'dangling-hyphen,line-too-long', ] + - repo: https://github.com/golangci/golangci-lint + rev: v1.59.0 + hooks: + # no built-in support for multiple go.mod + # https://github.com/golangci/golangci-lint/issues/828 + - id: golangci-lint-full + name: golangci-lint-full-arrow + entry: bash -c 'cd go/arrow && golangci-lint run' + - id: golangci-lint-full + name: golangci-lint-full-parquet + entry: bash -c 'cd go/parquet && golangci-lint run' + - id: golangci-lint-full + name: golangci-lint-full-internal + entry: bash -c 'cd go/internal && golangci-lint run' diff --git a/go/arrow/array/bufferbuilder_numeric_test.go b/go/arrow/array/bufferbuilder_numeric_test.go index df48dcff2e70f..372ba6976269d 100644 --- a/go/arrow/array/bufferbuilder_numeric_test.go +++ b/go/arrow/array/bufferbuilder_numeric_test.go @@ -20,8 +20,8 @@ import ( "testing" "unsafe" - "github.com/apache/arrow/go/v17/arrow/memory" "github.com/apache/arrow/go/v17/arrow/endian" + "github.com/apache/arrow/go/v17/arrow/memory" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/array/numeric.gen.go b/go/arrow/array/numeric.gen.go index 1d65657c5fae8..b962cda40b8b3 100644 --- a/go/arrow/array/numeric.gen.go +++ b/go/arrow/array/numeric.gen.go @@ -307,7 +307,7 @@ func (a *Float64) MarshalJSON() ([]byte, error) { default: vals[i] = f } - + } return json.Marshal(vals) diff --git a/go/arrow/array/numeric_test.go b/go/arrow/array/numeric_test.go index f775035c66652..3013d45acbb2b 100644 --- a/go/arrow/array/numeric_test.go +++ b/go/arrow/array/numeric_test.go @@ -16,7 +16,7 @@ package array_test -import ( +import ( "math" "reflect" "testing" @@ -144,7 +144,7 @@ func TestFloat16MarshalJSON(t *testing.T) { bldr := array.NewFloat16Builder(pool) defer bldr.Release() - + jsonstr := `[0, 1, 2, 3, "NaN", "NaN", 4, 5, "+Inf", "-Inf"]` bldr.Append(float16.New(0)) @@ -158,7 +158,6 @@ func TestFloat16MarshalJSON(t *testing.T) { bldr.Append(float16.Inf()) bldr.Append(float16.Inf().Negate()) - expected := bldr.NewFloat16Array() defer expected.Release() expected_json, err := expected.MarshalJSON() @@ -172,7 +171,7 @@ func TestFloat32MarshalJSON(t *testing.T) { bldr := array.NewFloat32Builder(pool) defer bldr.Release() - + jsonstr := `[0, 1, "+Inf", 2, 3, "NaN", "NaN", 4, 5, "-Inf"]` bldr.Append(0) @@ -186,10 +185,9 @@ func TestFloat32MarshalJSON(t *testing.T) { bldr.Append(5) bldr.Append(float32(math.Inf(-1))) - expected := bldr.NewFloat32Array() defer expected.Release() - + expected_json, err := expected.MarshalJSON() assert.NoError(t, err) @@ -223,7 +221,7 @@ func TestFloat64MarshalJSON(t *testing.T) { assert.NoError(t, err) assert.JSONEq(t, jsonstr, string(expected_json)) - + } func TestUnmarshalSpecialFloat(t *testing.T) { diff --git a/go/arrow/array/numericbuilder.gen_test.go b/go/arrow/array/numericbuilder.gen_test.go index 43b14c1868666..b43aa7f807090 100644 --- a/go/arrow/array/numericbuilder.gen_test.go +++ b/go/arrow/array/numericbuilder.gen_test.go @@ -648,9 +648,9 @@ func TestFloat64BuilderUnmarshalJSON(t *testing.T) { arr := bldr.NewFloat64Array() defer arr.Release() - + assert.NotNil(t, arr) - + assert.False(t, math.IsInf(float64(arr.Value(0)), 0), arr.Value(0)) assert.True(t, math.IsInf(float64(arr.Value(2)), 1), arr.Value(2)) assert.True(t, math.IsNaN(float64(arr.Value(5))), arr.Value(5)) @@ -1276,9 +1276,9 @@ func TestFloat32BuilderUnmarshalJSON(t *testing.T) { arr := bldr.NewFloat32Array() defer arr.Release() - + assert.NotNil(t, arr) - + assert.False(t, math.IsInf(float64(arr.Value(0)), 0), arr.Value(0)) assert.True(t, math.IsInf(float64(arr.Value(2)), 1), arr.Value(2)) assert.True(t, math.IsNaN(float64(arr.Value(5))), arr.Value(5)) diff --git a/go/arrow/array/record_test.go b/go/arrow/array/record_test.go index 36bb0eaa4c511..be6a26eb1a6ba 100644 --- a/go/arrow/array/record_test.go +++ b/go/arrow/array/record_test.go @@ -94,7 +94,7 @@ func TestRecord(t *testing.T) { if _, err := rec.SetColumn(0, col2_1); err == nil { t.Fatalf("expected an error") } - newRec, err := rec.SetColumn(1, col2_1); + newRec, err := rec.SetColumn(1, col2_1) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/go/arrow/datatype_nested_test.go b/go/arrow/datatype_nested_test.go index a1daa8e58df31..fc4c672c6b768 100644 --- a/go/arrow/datatype_nested_test.go +++ b/go/arrow/datatype_nested_test.go @@ -94,14 +94,14 @@ func TestStructOf(t *testing.T) { fields: []Field{{Name: "f1", Type: PrimitiveTypes.Int32}}, want: &StructType{ fields: []Field{{Name: "f1", Type: PrimitiveTypes.Int32}}, - index: map[string][]int{"f1": []int{0}}, + index: map[string][]int{"f1": {0}}, }, }, { fields: []Field{{Name: "f1", Type: PrimitiveTypes.Int32, Nullable: true}}, want: &StructType{ fields: []Field{{Name: "f1", Type: PrimitiveTypes.Int32, Nullable: true}}, - index: map[string][]int{"f1": []int{0}}, + index: map[string][]int{"f1": {0}}, }, }, { @@ -114,7 +114,7 @@ func TestStructOf(t *testing.T) { {Name: "f1", Type: PrimitiveTypes.Int32}, {Name: "", Type: PrimitiveTypes.Int64}, }, - index: map[string][]int{"f1": []int{0}, "": []int{1}}, + index: map[string][]int{"f1": {0}, "": {1}}, }, }, { @@ -127,7 +127,7 @@ func TestStructOf(t *testing.T) { {Name: "f1", Type: PrimitiveTypes.Int32}, {Name: "f2", Type: PrimitiveTypes.Int64}, }, - index: map[string][]int{"f1": []int{0}, "f2": []int{1}}, + index: map[string][]int{"f1": {0}, "f2": {1}}, }, }, { @@ -142,7 +142,7 @@ func TestStructOf(t *testing.T) { {Name: "f2", Type: PrimitiveTypes.Int64}, {Name: "f3", Type: ListOf(PrimitiveTypes.Float64)}, }, - index: map[string][]int{"f1": []int{0}, "f2": []int{1}, "f3": []int{2}}, + index: map[string][]int{"f1": {0}, "f2": {1}, "f3": {2}}, }, }, { @@ -157,7 +157,7 @@ func TestStructOf(t *testing.T) { {Name: "f2", Type: PrimitiveTypes.Int64}, {Name: "f3", Type: ListOf(ListOf(PrimitiveTypes.Float64))}, }, - index: map[string][]int{"f1": []int{0}, "f2": []int{1}, "f3": []int{2}}, + index: map[string][]int{"f1": {0}, "f2": {1}, "f3": {2}}, }, }, { @@ -172,7 +172,7 @@ func TestStructOf(t *testing.T) { {Name: "f2", Type: PrimitiveTypes.Int64}, {Name: "f3", Type: ListOf(ListOf(StructOf(Field{Name: "f1", Type: PrimitiveTypes.Float64})))}, }, - index: map[string][]int{"f1": []int{0}, "f2": []int{1}, "f3": []int{2}}, + index: map[string][]int{"f1": {0}, "f2": {1}, "f3": {2}}, }, }, { @@ -187,7 +187,7 @@ func TestStructOf(t *testing.T) { {Name: "f2", Type: PrimitiveTypes.Int64}, {Name: "f1", Type: PrimitiveTypes.Int64}, }, - index: map[string][]int{"f1": []int{0, 2}, "f2": []int{1}}, + index: map[string][]int{"f1": {0, 2}, "f2": {1}}, }, }, } { diff --git a/go/arrow/flight/gen/flight/Flight.pb.go b/go/arrow/flight/gen/flight/Flight.pb.go index d9477ee062fa8..ea35f469116ab 100644 --- a/go/arrow/flight/gen/flight/Flight.pb.go +++ b/go/arrow/flight/gen/flight/Flight.pb.go @@ -24,11 +24,12 @@ package flight import ( + reflect "reflect" + sync "sync" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" timestamppb "google.golang.org/protobuf/types/known/timestamppb" - reflect "reflect" - sync "sync" ) const ( diff --git a/go/arrow/flight/gen/flight/FlightSql.pb.go b/go/arrow/flight/gen/flight/FlightSql.pb.go index 196c1d6b33643..f8f5e17d76bd2 100644 --- a/go/arrow/flight/gen/flight/FlightSql.pb.go +++ b/go/arrow/flight/gen/flight/FlightSql.pb.go @@ -24,11 +24,12 @@ package flight import ( + reflect "reflect" + sync "sync" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" descriptorpb "google.golang.org/protobuf/types/descriptorpb" - reflect "reflect" - sync "sync" ) const ( diff --git a/go/arrow/flight/gen/flight/Flight_grpc.pb.go b/go/arrow/flight/gen/flight/Flight_grpc.pb.go index 11bbb00131ddb..da5601b46ab95 100644 --- a/go/arrow/flight/gen/flight/Flight_grpc.pb.go +++ b/go/arrow/flight/gen/flight/Flight_grpc.pb.go @@ -8,6 +8,7 @@ package flight import ( context "context" + grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" diff --git a/go/arrow/float16/float16.go b/go/arrow/float16/float16.go index ecf5c9ddce9db..f61db40ef498c 100644 --- a/go/arrow/float16/float16.go +++ b/go/arrow/float16/float16.go @@ -175,7 +175,7 @@ func (n Num) Signbit() bool { return (n.bits & 0x8000) != 0 } func (n Num) IsNaN() bool { return (n.bits & 0x7fff) > 0x7c00 } -func (n Num) IsInf() bool {return (n.bits & 0x7c00) == 0x7c00 } +func (n Num) IsInf() bool { return (n.bits & 0x7c00) == 0x7c00 } func (n Num) IsZero() bool { return (n.bits & 0x7fff) == 0 } diff --git a/go/arrow/gen-flatbuffers.go b/go/arrow/gen-flatbuffers.go index 5c8eba4a24757..720016e0bf168 100644 --- a/go/arrow/gen-flatbuffers.go +++ b/go/arrow/gen-flatbuffers.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build ignore // +build ignore package main diff --git a/go/arrow/internal/debug/assert_off.go b/go/arrow/internal/debug/assert_off.go index 52b9a233169d2..1450ecc98a26e 100644 --- a/go/arrow/internal/debug/assert_off.go +++ b/go/arrow/internal/debug/assert_off.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !assert // +build !assert package debug diff --git a/go/arrow/internal/debug/assert_on.go b/go/arrow/internal/debug/assert_on.go index 2aa5d6ace4cf0..4a57169b31358 100644 --- a/go/arrow/internal/debug/assert_on.go +++ b/go/arrow/internal/debug/assert_on.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build assert // +build assert package debug diff --git a/go/arrow/internal/debug/doc.go b/go/arrow/internal/debug/doc.go index 3ee1783ca4bda..094e427a22e09 100644 --- a/go/arrow/internal/debug/doc.go +++ b/go/arrow/internal/debug/doc.go @@ -17,14 +17,12 @@ /* Package debug provides APIs for conditional runtime assertions and debug logging. - -Using Assert +# Using Assert To enable runtime assertions, build with the assert tag. When the assert tag is omitted, the code for the assertion will be omitted from the binary. - -Using Log +# Using Log To enable runtime debug logs, build with the debug tag. When the debug tag is omitted, the code for logging will be omitted from the binary. diff --git a/go/arrow/internal/debug/log_off.go b/go/arrow/internal/debug/log_off.go index 48da8e1ee94c7..760a5cdc0dc01 100644 --- a/go/arrow/internal/debug/log_off.go +++ b/go/arrow/internal/debug/log_off.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !debug // +build !debug package debug diff --git a/go/arrow/internal/debug/log_on.go b/go/arrow/internal/debug/log_on.go index 99d0c8ae33fef..2588e7d1069f0 100644 --- a/go/arrow/internal/debug/log_on.go +++ b/go/arrow/internal/debug/log_on.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build debug // +build debug package debug diff --git a/go/arrow/internal/debug/util.go b/go/arrow/internal/debug/util.go index 7bd3d5389e669..ea4eba7fb5cb8 100644 --- a/go/arrow/internal/debug/util.go +++ b/go/arrow/internal/debug/util.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build debug || assert // +build debug assert package debug diff --git a/go/arrow/internal/flatbuf/Binary.go b/go/arrow/internal/flatbuf/Binary.go index e8018e74c4151..95e015595b548 100644 --- a/go/arrow/internal/flatbuf/Binary.go +++ b/go/arrow/internal/flatbuf/Binary.go @@ -22,7 +22,7 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// Opaque binary data +// / Opaque binary data type Binary struct { _tab flatbuffers.Table } diff --git a/go/arrow/internal/flatbuf/BinaryView.go b/go/arrow/internal/flatbuf/BinaryView.go index 09ca5e7db9601..f6906674bdbc7 100644 --- a/go/arrow/internal/flatbuf/BinaryView.go +++ b/go/arrow/internal/flatbuf/BinaryView.go @@ -22,13 +22,13 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// Logically the same as Binary, but the internal representation uses a view -/// struct that contains the string length and either the string's entire data -/// inline (for small strings) or an inlined prefix, an index of another buffer, -/// and an offset pointing to a slice in that buffer (for non-small strings). -/// -/// Since it uses a variable number of data buffers, each Field with this type -/// must have a corresponding entry in `variadicBufferCounts`. +// / Logically the same as Binary, but the internal representation uses a view +// / struct that contains the string length and either the string's entire data +// / inline (for small strings) or an inlined prefix, an index of another buffer, +// / and an offset pointing to a slice in that buffer (for non-small strings). +// / +// / Since it uses a variable number of data buffers, each Field with this type +// / must have a corresponding entry in `variadicBufferCounts`. type BinaryView struct { _tab flatbuffers.Table } diff --git a/go/arrow/internal/flatbuf/Block.go b/go/arrow/internal/flatbuf/Block.go index 57a697b196883..8e33d3e641543 100644 --- a/go/arrow/internal/flatbuf/Block.go +++ b/go/arrow/internal/flatbuf/Block.go @@ -35,31 +35,34 @@ func (rcv *Block) Table() flatbuffers.Table { return rcv._tab.Table } -/// Index to the start of the RecordBlock (note this is past the Message header) +// / Index to the start of the RecordBlock (note this is past the Message header) func (rcv *Block) Offset() int64 { return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(0)) } -/// Index to the start of the RecordBlock (note this is past the Message header) + +// / Index to the start of the RecordBlock (note this is past the Message header) func (rcv *Block) MutateOffset(n int64) bool { return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(0), n) } -/// Length of the metadata +// / Length of the metadata func (rcv *Block) MetaDataLength() int32 { return rcv._tab.GetInt32(rcv._tab.Pos + flatbuffers.UOffsetT(8)) } -/// Length of the metadata + +// / Length of the metadata func (rcv *Block) MutateMetaDataLength(n int32) bool { return rcv._tab.MutateInt32(rcv._tab.Pos+flatbuffers.UOffsetT(8), n) } -/// Length of the data (this is aligned so there can be a gap between this and -/// the metadata). +// / Length of the data (this is aligned so there can be a gap between this and +// / the metadata). func (rcv *Block) BodyLength() int64 { return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(16)) } -/// Length of the data (this is aligned so there can be a gap between this and -/// the metadata). + +// / Length of the data (this is aligned so there can be a gap between this and +// / the metadata). func (rcv *Block) MutateBodyLength(n int64) bool { return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(16), n) } diff --git a/go/arrow/internal/flatbuf/BodyCompression.go b/go/arrow/internal/flatbuf/BodyCompression.go index 6468e23135254..c23c29190216b 100644 --- a/go/arrow/internal/flatbuf/BodyCompression.go +++ b/go/arrow/internal/flatbuf/BodyCompression.go @@ -22,9 +22,9 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// Optional compression for the memory buffers constituting IPC message -/// bodies. Intended for use with RecordBatch but could be used for other -/// message types +// / Optional compression for the memory buffers constituting IPC message +// / bodies. Intended for use with RecordBatch but could be used for other +// / message types type BodyCompression struct { _tab flatbuffers.Table } @@ -45,8 +45,8 @@ func (rcv *BodyCompression) Table() flatbuffers.Table { return rcv._tab } -/// Compressor library. -/// For LZ4_FRAME, each compressed buffer must consist of a single frame. +// / Compressor library. +// / For LZ4_FRAME, each compressed buffer must consist of a single frame. func (rcv *BodyCompression) Codec() CompressionType { o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) if o != 0 { @@ -55,13 +55,13 @@ func (rcv *BodyCompression) Codec() CompressionType { return 0 } -/// Compressor library. -/// For LZ4_FRAME, each compressed buffer must consist of a single frame. +// / Compressor library. +// / For LZ4_FRAME, each compressed buffer must consist of a single frame. func (rcv *BodyCompression) MutateCodec(n CompressionType) bool { return rcv._tab.MutateInt8Slot(4, int8(n)) } -/// Indicates the way the record batch body was compressed +// / Indicates the way the record batch body was compressed func (rcv *BodyCompression) Method() BodyCompressionMethod { o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) if o != 0 { @@ -70,7 +70,7 @@ func (rcv *BodyCompression) Method() BodyCompressionMethod { return 0 } -/// Indicates the way the record batch body was compressed +// / Indicates the way the record batch body was compressed func (rcv *BodyCompression) MutateMethod(n BodyCompressionMethod) bool { return rcv._tab.MutateInt8Slot(6, int8(n)) } diff --git a/go/arrow/internal/flatbuf/BodyCompressionMethod.go b/go/arrow/internal/flatbuf/BodyCompressionMethod.go index 108ab3e07fba6..bb7234b3989b5 100644 --- a/go/arrow/internal/flatbuf/BodyCompressionMethod.go +++ b/go/arrow/internal/flatbuf/BodyCompressionMethod.go @@ -20,9 +20,9 @@ package flatbuf import "strconv" -/// Provided for forward compatibility in case we need to support different -/// strategies for compressing the IPC message body (like whole-body -/// compression rather than buffer-level) in the future +// / Provided for forward compatibility in case we need to support different +// / strategies for compressing the IPC message body (like whole-body +// / compression rather than buffer-level) in the future type BodyCompressionMethod int8 const ( diff --git a/go/arrow/internal/flatbuf/Buffer.go b/go/arrow/internal/flatbuf/Buffer.go index eba8d99b28e9b..e650e06a57026 100644 --- a/go/arrow/internal/flatbuf/Buffer.go +++ b/go/arrow/internal/flatbuf/Buffer.go @@ -22,8 +22,8 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// ---------------------------------------------------------------------- -/// A Buffer represents a single contiguous memory segment +// / ---------------------------------------------------------------------- +// / A Buffer represents a single contiguous memory segment type Buffer struct { _tab flatbuffers.Struct } @@ -37,30 +37,32 @@ func (rcv *Buffer) Table() flatbuffers.Table { return rcv._tab.Table } -/// The relative offset into the shared memory page where the bytes for this -/// buffer starts +// / The relative offset into the shared memory page where the bytes for this +// / buffer starts func (rcv *Buffer) Offset() int64 { return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(0)) } -/// The relative offset into the shared memory page where the bytes for this -/// buffer starts + +// / The relative offset into the shared memory page where the bytes for this +// / buffer starts func (rcv *Buffer) MutateOffset(n int64) bool { return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(0), n) } -/// The absolute length (in bytes) of the memory buffer. The memory is found -/// from offset (inclusive) to offset + length (non-inclusive). When building -/// messages using the encapsulated IPC message, padding bytes may be written -/// after a buffer, but such padding bytes do not need to be accounted for in -/// the size here. +// / The absolute length (in bytes) of the memory buffer. The memory is found +// / from offset (inclusive) to offset + length (non-inclusive). When building +// / messages using the encapsulated IPC message, padding bytes may be written +// / after a buffer, but such padding bytes do not need to be accounted for in +// / the size here. func (rcv *Buffer) Length() int64 { return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(8)) } -/// The absolute length (in bytes) of the memory buffer. The memory is found -/// from offset (inclusive) to offset + length (non-inclusive). When building -/// messages using the encapsulated IPC message, padding bytes may be written -/// after a buffer, but such padding bytes do not need to be accounted for in -/// the size here. + +// / The absolute length (in bytes) of the memory buffer. The memory is found +// / from offset (inclusive) to offset + length (non-inclusive). When building +// / messages using the encapsulated IPC message, padding bytes may be written +// / after a buffer, but such padding bytes do not need to be accounted for in +// / the size here. func (rcv *Buffer) MutateLength(n int64) bool { return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(8), n) } diff --git a/go/arrow/internal/flatbuf/Date.go b/go/arrow/internal/flatbuf/Date.go index 32983ec54ccc2..985a8f79955a4 100644 --- a/go/arrow/internal/flatbuf/Date.go +++ b/go/arrow/internal/flatbuf/Date.go @@ -22,12 +22,12 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// Date is either a 32-bit or 64-bit signed integer type representing an -/// elapsed time since UNIX epoch (1970-01-01), stored in either of two units: -/// -/// * Milliseconds (64 bits) indicating UNIX time elapsed since the epoch (no -/// leap seconds), where the values are evenly divisible by 86400000 -/// * Days (32 bits) since the UNIX epoch +// / Date is either a 32-bit or 64-bit signed integer type representing an +// / elapsed time since UNIX epoch (1970-01-01), stored in either of two units: +// / +// / * Milliseconds (64 bits) indicating UNIX time elapsed since the epoch (no +// / leap seconds), where the values are evenly divisible by 86400000 +// / * Days (32 bits) since the UNIX epoch type Date struct { _tab flatbuffers.Table } diff --git a/go/arrow/internal/flatbuf/Decimal.go b/go/arrow/internal/flatbuf/Decimal.go index c9de254d1dcbd..2fc9d5ad6586c 100644 --- a/go/arrow/internal/flatbuf/Decimal.go +++ b/go/arrow/internal/flatbuf/Decimal.go @@ -22,10 +22,10 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// Exact decimal value represented as an integer value in two's -/// complement. Currently only 128-bit (16-byte) and 256-bit (32-byte) integers -/// are used. The representation uses the endianness indicated -/// in the Schema. +// / Exact decimal value represented as an integer value in two's +// / complement. Currently only 128-bit (16-byte) and 256-bit (32-byte) integers +// / are used. The representation uses the endianness indicated +// / in the Schema. type Decimal struct { _tab flatbuffers.Table } @@ -46,7 +46,7 @@ func (rcv *Decimal) Table() flatbuffers.Table { return rcv._tab } -/// Total number of decimal digits +// / Total number of decimal digits func (rcv *Decimal) Precision() int32 { o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) if o != 0 { @@ -55,12 +55,12 @@ func (rcv *Decimal) Precision() int32 { return 0 } -/// Total number of decimal digits +// / Total number of decimal digits func (rcv *Decimal) MutatePrecision(n int32) bool { return rcv._tab.MutateInt32Slot(4, n) } -/// Number of digits after the decimal point "." +// / Number of digits after the decimal point "." func (rcv *Decimal) Scale() int32 { o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) if o != 0 { @@ -69,13 +69,13 @@ func (rcv *Decimal) Scale() int32 { return 0 } -/// Number of digits after the decimal point "." +// / Number of digits after the decimal point "." func (rcv *Decimal) MutateScale(n int32) bool { return rcv._tab.MutateInt32Slot(6, n) } -/// Number of bits per value. The only accepted widths are 128 and 256. -/// We use bitWidth for consistency with Int::bitWidth. +// / Number of bits per value. The only accepted widths are 128 and 256. +// / We use bitWidth for consistency with Int::bitWidth. func (rcv *Decimal) BitWidth() int32 { o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) if o != 0 { @@ -84,8 +84,8 @@ func (rcv *Decimal) BitWidth() int32 { return 128 } -/// Number of bits per value. The only accepted widths are 128 and 256. -/// We use bitWidth for consistency with Int::bitWidth. +// / Number of bits per value. The only accepted widths are 128 and 256. +// / We use bitWidth for consistency with Int::bitWidth. func (rcv *Decimal) MutateBitWidth(n int32) bool { return rcv._tab.MutateInt32Slot(8, n) } diff --git a/go/arrow/internal/flatbuf/DictionaryBatch.go b/go/arrow/internal/flatbuf/DictionaryBatch.go index 25b5384e46a5c..999c5fda46384 100644 --- a/go/arrow/internal/flatbuf/DictionaryBatch.go +++ b/go/arrow/internal/flatbuf/DictionaryBatch.go @@ -22,12 +22,12 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// For sending dictionary encoding information. Any Field can be -/// dictionary-encoded, but in this case none of its children may be -/// dictionary-encoded. -/// There is one vector / column per dictionary, but that vector / column -/// may be spread across multiple dictionary batches by using the isDelta -/// flag +// / For sending dictionary encoding information. Any Field can be +// / dictionary-encoded, but in this case none of its children may be +// / dictionary-encoded. +// / There is one vector / column per dictionary, but that vector / column +// / may be spread across multiple dictionary batches by using the isDelta +// / flag type DictionaryBatch struct { _tab flatbuffers.Table } @@ -73,9 +73,9 @@ func (rcv *DictionaryBatch) Data(obj *RecordBatch) *RecordBatch { return nil } -/// If isDelta is true the values in the dictionary are to be appended to a -/// dictionary with the indicated id. If isDelta is false this dictionary -/// should replace the existing dictionary. +// / If isDelta is true the values in the dictionary are to be appended to a +// / dictionary with the indicated id. If isDelta is false this dictionary +// / should replace the existing dictionary. func (rcv *DictionaryBatch) IsDelta() bool { o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) if o != 0 { @@ -84,9 +84,9 @@ func (rcv *DictionaryBatch) IsDelta() bool { return false } -/// If isDelta is true the values in the dictionary are to be appended to a -/// dictionary with the indicated id. If isDelta is false this dictionary -/// should replace the existing dictionary. +// / If isDelta is true the values in the dictionary are to be appended to a +// / dictionary with the indicated id. If isDelta is false this dictionary +// / should replace the existing dictionary. func (rcv *DictionaryBatch) MutateIsDelta(n bool) bool { return rcv._tab.MutateBoolSlot(8, n) } diff --git a/go/arrow/internal/flatbuf/DictionaryEncoding.go b/go/arrow/internal/flatbuf/DictionaryEncoding.go index a9b09530b2a52..44c3874219f1c 100644 --- a/go/arrow/internal/flatbuf/DictionaryEncoding.go +++ b/go/arrow/internal/flatbuf/DictionaryEncoding.go @@ -42,9 +42,9 @@ func (rcv *DictionaryEncoding) Table() flatbuffers.Table { return rcv._tab } -/// The known dictionary id in the application where this data is used. In -/// the file or streaming formats, the dictionary ids are found in the -/// DictionaryBatch messages +// / The known dictionary id in the application where this data is used. In +// / the file or streaming formats, the dictionary ids are found in the +// / DictionaryBatch messages func (rcv *DictionaryEncoding) Id() int64 { o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) if o != 0 { @@ -53,18 +53,18 @@ func (rcv *DictionaryEncoding) Id() int64 { return 0 } -/// The known dictionary id in the application where this data is used. In -/// the file or streaming formats, the dictionary ids are found in the -/// DictionaryBatch messages +// / The known dictionary id in the application where this data is used. In +// / the file or streaming formats, the dictionary ids are found in the +// / DictionaryBatch messages func (rcv *DictionaryEncoding) MutateId(n int64) bool { return rcv._tab.MutateInt64Slot(4, n) } -/// The dictionary indices are constrained to be non-negative integers. If -/// this field is null, the indices must be signed int32. To maximize -/// cross-language compatibility and performance, implementations are -/// recommended to prefer signed integer types over unsigned integer types -/// and to avoid uint64 indices unless they are required by an application. +// / The dictionary indices are constrained to be non-negative integers. If +// / this field is null, the indices must be signed int32. To maximize +// / cross-language compatibility and performance, implementations are +// / recommended to prefer signed integer types over unsigned integer types +// / and to avoid uint64 indices unless they are required by an application. func (rcv *DictionaryEncoding) IndexType(obj *Int) *Int { o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) if o != 0 { @@ -78,15 +78,15 @@ func (rcv *DictionaryEncoding) IndexType(obj *Int) *Int { return nil } -/// The dictionary indices are constrained to be non-negative integers. If -/// this field is null, the indices must be signed int32. To maximize -/// cross-language compatibility and performance, implementations are -/// recommended to prefer signed integer types over unsigned integer types -/// and to avoid uint64 indices unless they are required by an application. -/// By default, dictionaries are not ordered, or the order does not have -/// semantic meaning. In some statistical, applications, dictionary-encoding -/// is used to represent ordered categorical data, and we provide a way to -/// preserve that metadata here +// / The dictionary indices are constrained to be non-negative integers. If +// / this field is null, the indices must be signed int32. To maximize +// / cross-language compatibility and performance, implementations are +// / recommended to prefer signed integer types over unsigned integer types +// / and to avoid uint64 indices unless they are required by an application. +// / By default, dictionaries are not ordered, or the order does not have +// / semantic meaning. In some statistical, applications, dictionary-encoding +// / is used to represent ordered categorical data, and we provide a way to +// / preserve that metadata here func (rcv *DictionaryEncoding) IsOrdered() bool { o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) if o != 0 { @@ -95,10 +95,10 @@ func (rcv *DictionaryEncoding) IsOrdered() bool { return false } -/// By default, dictionaries are not ordered, or the order does not have -/// semantic meaning. In some statistical, applications, dictionary-encoding -/// is used to represent ordered categorical data, and we provide a way to -/// preserve that metadata here +// / By default, dictionaries are not ordered, or the order does not have +// / semantic meaning. In some statistical, applications, dictionary-encoding +// / is used to represent ordered categorical data, and we provide a way to +// / preserve that metadata here func (rcv *DictionaryEncoding) MutateIsOrdered(n bool) bool { return rcv._tab.MutateBoolSlot(8, n) } diff --git a/go/arrow/internal/flatbuf/DictionaryKind.go b/go/arrow/internal/flatbuf/DictionaryKind.go index 126ba5f7f6bb0..6825100515612 100644 --- a/go/arrow/internal/flatbuf/DictionaryKind.go +++ b/go/arrow/internal/flatbuf/DictionaryKind.go @@ -20,11 +20,11 @@ package flatbuf import "strconv" -/// ---------------------------------------------------------------------- -/// Dictionary encoding metadata -/// Maintained for forwards compatibility, in the future -/// Dictionaries might be explicit maps between integers and values -/// allowing for non-contiguous index values +// / ---------------------------------------------------------------------- +// / Dictionary encoding metadata +// / Maintained for forwards compatibility, in the future +// / Dictionaries might be explicit maps between integers and values +// / allowing for non-contiguous index values type DictionaryKind int16 const ( diff --git a/go/arrow/internal/flatbuf/Endianness.go b/go/arrow/internal/flatbuf/Endianness.go index cefa2ff9c06ed..c9619b7b0d978 100644 --- a/go/arrow/internal/flatbuf/Endianness.go +++ b/go/arrow/internal/flatbuf/Endianness.go @@ -20,8 +20,8 @@ package flatbuf import "strconv" -/// ---------------------------------------------------------------------- -/// Endianness of the platform producing the data +// / ---------------------------------------------------------------------- +// / Endianness of the platform producing the data type Endianness int16 const ( diff --git a/go/arrow/internal/flatbuf/Feature.go b/go/arrow/internal/flatbuf/Feature.go index ae5a0398b607d..2204c440ed4fe 100644 --- a/go/arrow/internal/flatbuf/Feature.go +++ b/go/arrow/internal/flatbuf/Feature.go @@ -20,35 +20,35 @@ package flatbuf import "strconv" -/// Represents Arrow Features that might not have full support -/// within implementations. This is intended to be used in -/// two scenarios: -/// 1. A mechanism for readers of Arrow Streams -/// and files to understand that the stream or file makes -/// use of a feature that isn't supported or unknown to -/// the implementation (and therefore can meet the Arrow -/// forward compatibility guarantees). -/// 2. A means of negotiating between a client and server -/// what features a stream is allowed to use. The enums -/// values here are intented to represent higher level -/// features, additional details maybe negotiated -/// with key-value pairs specific to the protocol. -/// -/// Enums added to this list should be assigned power-of-two values -/// to facilitate exchanging and comparing bitmaps for supported -/// features. +// / Represents Arrow Features that might not have full support +// / within implementations. This is intended to be used in +// / two scenarios: +// / 1. A mechanism for readers of Arrow Streams +// / and files to understand that the stream or file makes +// / use of a feature that isn't supported or unknown to +// / the implementation (and therefore can meet the Arrow +// / forward compatibility guarantees). +// / 2. A means of negotiating between a client and server +// / what features a stream is allowed to use. The enums +// / values here are intented to represent higher level +// / features, additional details maybe negotiated +// / with key-value pairs specific to the protocol. +// / +// / Enums added to this list should be assigned power-of-two values +// / to facilitate exchanging and comparing bitmaps for supported +// / features. type Feature int64 const ( /// Needed to make flatbuffers happy. - FeatureUNUSED Feature = 0 + FeatureUNUSED Feature = 0 /// The stream makes use of multiple full dictionaries with the /// same ID and assumes clients implement dictionary replacement /// correctly. FeatureDICTIONARY_REPLACEMENT Feature = 1 /// The stream makes use of compressed bodies as described /// in Message.fbs. - FeatureCOMPRESSED_BODY Feature = 2 + FeatureCOMPRESSED_BODY Feature = 2 ) var EnumNamesFeature = map[Feature]string{ diff --git a/go/arrow/internal/flatbuf/Field.go b/go/arrow/internal/flatbuf/Field.go index c03cf2f878b6f..8aed29bc48137 100644 --- a/go/arrow/internal/flatbuf/Field.go +++ b/go/arrow/internal/flatbuf/Field.go @@ -22,9 +22,9 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// ---------------------------------------------------------------------- -/// A field represents a named column in a record / row batch or child of a -/// nested type. +// / ---------------------------------------------------------------------- +// / A field represents a named column in a record / row batch or child of a +// / nested type. type Field struct { _tab flatbuffers.Table } @@ -45,7 +45,7 @@ func (rcv *Field) Table() flatbuffers.Table { return rcv._tab } -/// Name is not required, in i.e. a List +// / Name is not required, in i.e. a List func (rcv *Field) Name() []byte { o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) if o != 0 { @@ -54,8 +54,8 @@ func (rcv *Field) Name() []byte { return nil } -/// Name is not required, in i.e. a List -/// Whether or not this field can contain nulls. Should be true in general. +// / Name is not required, in i.e. a List +// / Whether or not this field can contain nulls. Should be true in general. func (rcv *Field) Nullable() bool { o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) if o != 0 { @@ -64,7 +64,7 @@ func (rcv *Field) Nullable() bool { return false } -/// Whether or not this field can contain nulls. Should be true in general. +// / Whether or not this field can contain nulls. Should be true in general. func (rcv *Field) MutateNullable(n bool) bool { return rcv._tab.MutateBoolSlot(6, n) } @@ -81,7 +81,7 @@ func (rcv *Field) MutateTypeType(n Type) bool { return rcv._tab.MutateByteSlot(8, byte(n)) } -/// This is the type of the decoded value if the field is dictionary encoded. +// / This is the type of the decoded value if the field is dictionary encoded. func (rcv *Field) Type(obj *flatbuffers.Table) bool { o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) if o != 0 { @@ -91,8 +91,8 @@ func (rcv *Field) Type(obj *flatbuffers.Table) bool { return false } -/// This is the type of the decoded value if the field is dictionary encoded. -/// Present only if the field is dictionary encoded. +// / This is the type of the decoded value if the field is dictionary encoded. +// / Present only if the field is dictionary encoded. func (rcv *Field) Dictionary(obj *DictionaryEncoding) *DictionaryEncoding { o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) if o != 0 { @@ -106,9 +106,9 @@ func (rcv *Field) Dictionary(obj *DictionaryEncoding) *DictionaryEncoding { return nil } -/// Present only if the field is dictionary encoded. -/// children apply only to nested data types like Struct, List and Union. For -/// primitive types children will have length 0. +// / Present only if the field is dictionary encoded. +// / children apply only to nested data types like Struct, List and Union. For +// / primitive types children will have length 0. func (rcv *Field) Children(obj *Field, j int) bool { o := flatbuffers.UOffsetT(rcv._tab.Offset(14)) if o != 0 { @@ -129,9 +129,9 @@ func (rcv *Field) ChildrenLength() int { return 0 } -/// children apply only to nested data types like Struct, List and Union. For -/// primitive types children will have length 0. -/// User-defined metadata +// / children apply only to nested data types like Struct, List and Union. For +// / primitive types children will have length 0. +// / User-defined metadata func (rcv *Field) CustomMetadata(obj *KeyValue, j int) bool { o := flatbuffers.UOffsetT(rcv._tab.Offset(16)) if o != 0 { @@ -152,7 +152,7 @@ func (rcv *Field) CustomMetadataLength() int { return 0 } -/// User-defined metadata +// / User-defined metadata func FieldStart(builder *flatbuffers.Builder) { builder.StartObject(7) } diff --git a/go/arrow/internal/flatbuf/FieldNode.go b/go/arrow/internal/flatbuf/FieldNode.go index 606b30bfebbd2..0e258a3d2cde8 100644 --- a/go/arrow/internal/flatbuf/FieldNode.go +++ b/go/arrow/internal/flatbuf/FieldNode.go @@ -22,15 +22,15 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// ---------------------------------------------------------------------- -/// Data structures for describing a table row batch (a collection of -/// equal-length Arrow arrays) -/// Metadata about a field at some level of a nested type tree (but not -/// its children). -/// -/// For example, a List with values `[[1, 2, 3], null, [4], [5, 6], null]` -/// would have {length: 5, null_count: 2} for its List node, and {length: 6, -/// null_count: 0} for its Int16 node, as separate FieldNode structs +// / ---------------------------------------------------------------------- +// / Data structures for describing a table row batch (a collection of +// / equal-length Arrow arrays) +// / Metadata about a field at some level of a nested type tree (but not +// / its children). +// / +// / For example, a List with values `[[1, 2, 3], null, [4], [5, 6], null]` +// / would have {length: 5, null_count: 2} for its List node, and {length: 6, +// / null_count: 0} for its Int16 node, as separate FieldNode structs type FieldNode struct { _tab flatbuffers.Struct } @@ -44,26 +44,28 @@ func (rcv *FieldNode) Table() flatbuffers.Table { return rcv._tab.Table } -/// The number of value slots in the Arrow array at this level of a nested -/// tree +// / The number of value slots in the Arrow array at this level of a nested +// / tree func (rcv *FieldNode) Length() int64 { return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(0)) } -/// The number of value slots in the Arrow array at this level of a nested -/// tree + +// / The number of value slots in the Arrow array at this level of a nested +// / tree func (rcv *FieldNode) MutateLength(n int64) bool { return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(0), n) } -/// The number of observed nulls. Fields with null_count == 0 may choose not -/// to write their physical validity bitmap out as a materialized buffer, -/// instead setting the length of the bitmap buffer to 0. +// / The number of observed nulls. Fields with null_count == 0 may choose not +// / to write their physical validity bitmap out as a materialized buffer, +// / instead setting the length of the bitmap buffer to 0. func (rcv *FieldNode) NullCount() int64 { return rcv._tab.GetInt64(rcv._tab.Pos + flatbuffers.UOffsetT(8)) } -/// The number of observed nulls. Fields with null_count == 0 may choose not -/// to write their physical validity bitmap out as a materialized buffer, -/// instead setting the length of the bitmap buffer to 0. + +// / The number of observed nulls. Fields with null_count == 0 may choose not +// / to write their physical validity bitmap out as a materialized buffer, +// / instead setting the length of the bitmap buffer to 0. func (rcv *FieldNode) MutateNullCount(n int64) bool { return rcv._tab.MutateInt64(rcv._tab.Pos+flatbuffers.UOffsetT(8), n) } diff --git a/go/arrow/internal/flatbuf/FixedSizeBinary.go b/go/arrow/internal/flatbuf/FixedSizeBinary.go index 4e660d5077f71..2725dfb90b966 100644 --- a/go/arrow/internal/flatbuf/FixedSizeBinary.go +++ b/go/arrow/internal/flatbuf/FixedSizeBinary.go @@ -42,7 +42,7 @@ func (rcv *FixedSizeBinary) Table() flatbuffers.Table { return rcv._tab } -/// Number of bytes per value +// / Number of bytes per value func (rcv *FixedSizeBinary) ByteWidth() int32 { o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) if o != 0 { @@ -51,7 +51,7 @@ func (rcv *FixedSizeBinary) ByteWidth() int32 { return 0 } -/// Number of bytes per value +// / Number of bytes per value func (rcv *FixedSizeBinary) MutateByteWidth(n int32) bool { return rcv._tab.MutateInt32Slot(4, n) } diff --git a/go/arrow/internal/flatbuf/FixedSizeList.go b/go/arrow/internal/flatbuf/FixedSizeList.go index dabf5cc8581da..534ca27f2fe21 100644 --- a/go/arrow/internal/flatbuf/FixedSizeList.go +++ b/go/arrow/internal/flatbuf/FixedSizeList.go @@ -42,7 +42,7 @@ func (rcv *FixedSizeList) Table() flatbuffers.Table { return rcv._tab } -/// Number of list items per value +// / Number of list items per value func (rcv *FixedSizeList) ListSize() int32 { o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) if o != 0 { @@ -51,7 +51,7 @@ func (rcv *FixedSizeList) ListSize() int32 { return 0 } -/// Number of list items per value +// / Number of list items per value func (rcv *FixedSizeList) MutateListSize(n int32) bool { return rcv._tab.MutateInt32Slot(4, n) } diff --git a/go/arrow/internal/flatbuf/Footer.go b/go/arrow/internal/flatbuf/Footer.go index 65b0ff0954614..d65af41e7f62e 100644 --- a/go/arrow/internal/flatbuf/Footer.go +++ b/go/arrow/internal/flatbuf/Footer.go @@ -22,9 +22,9 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// ---------------------------------------------------------------------- -/// Arrow File metadata -/// +// / ---------------------------------------------------------------------- +// / Arrow File metadata +// / type Footer struct { _tab flatbuffers.Table } @@ -108,7 +108,7 @@ func (rcv *Footer) RecordBatchesLength() int { return 0 } -/// User-defined metadata +// / User-defined metadata func (rcv *Footer) CustomMetadata(obj *KeyValue, j int) bool { o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) if o != 0 { @@ -129,7 +129,7 @@ func (rcv *Footer) CustomMetadataLength() int { return 0 } -/// User-defined metadata +// / User-defined metadata func FooterStart(builder *flatbuffers.Builder) { builder.StartObject(5) } diff --git a/go/arrow/internal/flatbuf/KeyValue.go b/go/arrow/internal/flatbuf/KeyValue.go index c1b85318ecd5f..0cd5dc62923e3 100644 --- a/go/arrow/internal/flatbuf/KeyValue.go +++ b/go/arrow/internal/flatbuf/KeyValue.go @@ -22,9 +22,9 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// ---------------------------------------------------------------------- -/// user defined key value pairs to add custom metadata to arrow -/// key namespacing is the responsibility of the user +// / ---------------------------------------------------------------------- +// / user defined key value pairs to add custom metadata to arrow +// / key namespacing is the responsibility of the user type KeyValue struct { _tab flatbuffers.Table } diff --git a/go/arrow/internal/flatbuf/LargeBinary.go b/go/arrow/internal/flatbuf/LargeBinary.go index 2c3befcc16fb9..b25ecc41aff51 100644 --- a/go/arrow/internal/flatbuf/LargeBinary.go +++ b/go/arrow/internal/flatbuf/LargeBinary.go @@ -22,8 +22,8 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// Same as Binary, but with 64-bit offsets, allowing to represent -/// extremely large data values. +// / Same as Binary, but with 64-bit offsets, allowing to represent +// / extremely large data values. type LargeBinary struct { _tab flatbuffers.Table } diff --git a/go/arrow/internal/flatbuf/LargeList.go b/go/arrow/internal/flatbuf/LargeList.go index 92f2284587445..d8bfb9c07df76 100644 --- a/go/arrow/internal/flatbuf/LargeList.go +++ b/go/arrow/internal/flatbuf/LargeList.go @@ -22,8 +22,8 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// Same as List, but with 64-bit offsets, allowing to represent -/// extremely large data values. +// / Same as List, but with 64-bit offsets, allowing to represent +// / extremely large data values. type LargeList struct { _tab flatbuffers.Table } diff --git a/go/arrow/internal/flatbuf/LargeListView.go b/go/arrow/internal/flatbuf/LargeListView.go index 5b1df149cd1e2..4608c1dec53d8 100644 --- a/go/arrow/internal/flatbuf/LargeListView.go +++ b/go/arrow/internal/flatbuf/LargeListView.go @@ -22,8 +22,8 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// Same as ListView, but with 64-bit offsets and sizes, allowing to represent -/// extremely large data values. +// / Same as ListView, but with 64-bit offsets and sizes, allowing to represent +// / extremely large data values. type LargeListView struct { _tab flatbuffers.Table } diff --git a/go/arrow/internal/flatbuf/LargeUtf8.go b/go/arrow/internal/flatbuf/LargeUtf8.go index e78b33e110066..4478fed856e6d 100644 --- a/go/arrow/internal/flatbuf/LargeUtf8.go +++ b/go/arrow/internal/flatbuf/LargeUtf8.go @@ -22,8 +22,8 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// Same as Utf8, but with 64-bit offsets, allowing to represent -/// extremely large data values. +// / Same as Utf8, but with 64-bit offsets, allowing to represent +// / extremely large data values. type LargeUtf8 struct { _tab flatbuffers.Table } diff --git a/go/arrow/internal/flatbuf/ListView.go b/go/arrow/internal/flatbuf/ListView.go index 46b1e0b3cbf2f..cde43cf5b6893 100644 --- a/go/arrow/internal/flatbuf/ListView.go +++ b/go/arrow/internal/flatbuf/ListView.go @@ -22,9 +22,9 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// Represents the same logical types that List can, but contains offsets and -/// sizes allowing for writes in any order and sharing of child values among -/// list values. +// / Represents the same logical types that List can, but contains offsets and +// / sizes allowing for writes in any order and sharing of child values among +// / list values. type ListView struct { _tab flatbuffers.Table } diff --git a/go/arrow/internal/flatbuf/Map.go b/go/arrow/internal/flatbuf/Map.go index 8802aba1ebd39..d4871e558199f 100644 --- a/go/arrow/internal/flatbuf/Map.go +++ b/go/arrow/internal/flatbuf/Map.go @@ -22,31 +22,31 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// A Map is a logical nested type that is represented as -/// -/// List> -/// -/// In this layout, the keys and values are each respectively contiguous. We do -/// not constrain the key and value types, so the application is responsible -/// for ensuring that the keys are hashable and unique. Whether the keys are sorted -/// may be set in the metadata for this field. -/// -/// In a field with Map type, the field has a child Struct field, which then -/// has two children: key type and the second the value type. The names of the -/// child fields may be respectively "entries", "key", and "value", but this is -/// not enforced. -/// -/// Map -/// ```text -/// - child[0] entries: Struct -/// - child[0] key: K -/// - child[1] value: V -/// ``` -/// Neither the "entries" field nor the "key" field may be nullable. -/// -/// The metadata is structured so that Arrow systems without special handling -/// for Map can make Map an alias for List. The "layout" attribute for the Map -/// field must have the same contents as a List. +// / A Map is a logical nested type that is represented as +// / +// / List> +// / +// / In this layout, the keys and values are each respectively contiguous. We do +// / not constrain the key and value types, so the application is responsible +// / for ensuring that the keys are hashable and unique. Whether the keys are sorted +// / may be set in the metadata for this field. +// / +// / In a field with Map type, the field has a child Struct field, which then +// / has two children: key type and the second the value type. The names of the +// / child fields may be respectively "entries", "key", and "value", but this is +// / not enforced. +// / +// / Map +// / ```text +// / - child[0] entries: Struct +// / - child[0] key: K +// / - child[1] value: V +// / ``` +// / Neither the "entries" field nor the "key" field may be nullable. +// / +// / The metadata is structured so that Arrow systems without special handling +// / for Map can make Map an alias for List. The "layout" attribute for the Map +// / field must have the same contents as a List. type Map struct { _tab flatbuffers.Table } @@ -67,7 +67,7 @@ func (rcv *Map) Table() flatbuffers.Table { return rcv._tab } -/// Set to true if the keys within each value are sorted +// / Set to true if the keys within each value are sorted func (rcv *Map) KeysSorted() bool { o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) if o != 0 { @@ -76,7 +76,7 @@ func (rcv *Map) KeysSorted() bool { return false } -/// Set to true if the keys within each value are sorted +// / Set to true if the keys within each value are sorted func (rcv *Map) MutateKeysSorted(n bool) bool { return rcv._tab.MutateBoolSlot(4, n) } diff --git a/go/arrow/internal/flatbuf/MessageHeader.go b/go/arrow/internal/flatbuf/MessageHeader.go index c12fc1058119d..d7f9907c7a7a2 100644 --- a/go/arrow/internal/flatbuf/MessageHeader.go +++ b/go/arrow/internal/flatbuf/MessageHeader.go @@ -20,14 +20,14 @@ package flatbuf import "strconv" -/// ---------------------------------------------------------------------- -/// The root Message type -/// This union enables us to easily send different message types without -/// redundant storage, and in the future we can easily add new message types. -/// -/// Arrow implementations do not need to implement all of the message types, -/// which may include experimental metadata types. For maximum compatibility, -/// it is best to send data using RecordBatch +// / ---------------------------------------------------------------------- +// / The root Message type +// / This union enables us to easily send different message types without +// / redundant storage, and in the future we can easily add new message types. +// / +// / Arrow implementations do not need to implement all of the message types, +// / which may include experimental metadata types. For maximum compatibility, +// / it is best to send data using RecordBatch type MessageHeader byte const ( diff --git a/go/arrow/internal/flatbuf/Null.go b/go/arrow/internal/flatbuf/Null.go index 3c3eb4bda3619..3b93a1b6ee965 100644 --- a/go/arrow/internal/flatbuf/Null.go +++ b/go/arrow/internal/flatbuf/Null.go @@ -22,7 +22,7 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// These are stored in the flatbuffer in the Type union below +// / These are stored in the flatbuffer in the Type union below type Null struct { _tab flatbuffers.Table } diff --git a/go/arrow/internal/flatbuf/RecordBatch.go b/go/arrow/internal/flatbuf/RecordBatch.go index c50f4a6e868ea..52c72a8a20ae4 100644 --- a/go/arrow/internal/flatbuf/RecordBatch.go +++ b/go/arrow/internal/flatbuf/RecordBatch.go @@ -22,9 +22,9 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// A data header describing the shared memory layout of a "record" or "row" -/// batch. Some systems call this a "row batch" internally and others a "record -/// batch". +// / A data header describing the shared memory layout of a "record" or "row" +// / batch. Some systems call this a "row batch" internally and others a "record +// / batch". type RecordBatch struct { _tab flatbuffers.Table } @@ -45,8 +45,8 @@ func (rcv *RecordBatch) Table() flatbuffers.Table { return rcv._tab } -/// number of records / rows. The arrays in the batch should all have this -/// length +// / number of records / rows. The arrays in the batch should all have this +// / length func (rcv *RecordBatch) Length() int64 { o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) if o != 0 { @@ -55,13 +55,13 @@ func (rcv *RecordBatch) Length() int64 { return 0 } -/// number of records / rows. The arrays in the batch should all have this -/// length +// / number of records / rows. The arrays in the batch should all have this +// / length func (rcv *RecordBatch) MutateLength(n int64) bool { return rcv._tab.MutateInt64Slot(4, n) } -/// Nodes correspond to the pre-ordered flattened logical schema +// / Nodes correspond to the pre-ordered flattened logical schema func (rcv *RecordBatch) Nodes(obj *FieldNode, j int) bool { o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) if o != 0 { @@ -81,13 +81,13 @@ func (rcv *RecordBatch) NodesLength() int { return 0 } -/// Nodes correspond to the pre-ordered flattened logical schema -/// Buffers correspond to the pre-ordered flattened buffer tree -/// -/// The number of buffers appended to this list depends on the schema. For -/// example, most primitive arrays will have 2 buffers, 1 for the validity -/// bitmap and 1 for the values. For struct arrays, there will only be a -/// single buffer for the validity (nulls) bitmap +// / Nodes correspond to the pre-ordered flattened logical schema +// / Buffers correspond to the pre-ordered flattened buffer tree +// / +// / The number of buffers appended to this list depends on the schema. For +// / example, most primitive arrays will have 2 buffers, 1 for the validity +// / bitmap and 1 for the values. For struct arrays, there will only be a +// / single buffer for the validity (nulls) bitmap func (rcv *RecordBatch) Buffers(obj *Buffer, j int) bool { o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) if o != 0 { @@ -107,13 +107,13 @@ func (rcv *RecordBatch) BuffersLength() int { return 0 } -/// Buffers correspond to the pre-ordered flattened buffer tree -/// -/// The number of buffers appended to this list depends on the schema. For -/// example, most primitive arrays will have 2 buffers, 1 for the validity -/// bitmap and 1 for the values. For struct arrays, there will only be a -/// single buffer for the validity (nulls) bitmap -/// Optional compression of the message body +// / Buffers correspond to the pre-ordered flattened buffer tree +// / +// / The number of buffers appended to this list depends on the schema. For +// / example, most primitive arrays will have 2 buffers, 1 for the validity +// / bitmap and 1 for the values. For struct arrays, there will only be a +// / single buffer for the validity (nulls) bitmap +// / Optional compression of the message body func (rcv *RecordBatch) Compression(obj *BodyCompression) *BodyCompression { o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) if o != 0 { @@ -127,21 +127,21 @@ func (rcv *RecordBatch) Compression(obj *BodyCompression) *BodyCompression { return nil } -/// Optional compression of the message body -/// Some types such as Utf8View are represented using a variable number of buffers. -/// For each such Field in the pre-ordered flattened logical schema, there will be -/// an entry in variadicBufferCounts to indicate the number of number of variadic -/// buffers which belong to that Field in the current RecordBatch. -/// -/// For example, the schema -/// col1: Struct -/// col2: Utf8View -/// contains two Fields with variadic buffers so variadicBufferCounts will have -/// two entries, the first counting the variadic buffers of `col1.beta` and the -/// second counting `col2`'s. -/// -/// This field may be omitted if and only if the schema contains no Fields with -/// a variable number of buffers, such as BinaryView and Utf8View. +// / Optional compression of the message body +// / Some types such as Utf8View are represented using a variable number of buffers. +// / For each such Field in the pre-ordered flattened logical schema, there will be +// / an entry in variadicBufferCounts to indicate the number of number of variadic +// / buffers which belong to that Field in the current RecordBatch. +// / +// / For example, the schema +// / col1: Struct +// / col2: Utf8View +// / contains two Fields with variadic buffers so variadicBufferCounts will have +// / two entries, the first counting the variadic buffers of `col1.beta` and the +// / second counting `col2`'s. +// / +// / This field may be omitted if and only if the schema contains no Fields with +// / a variable number of buffers, such as BinaryView and Utf8View. func (rcv *RecordBatch) VariadicBufferCounts(j int) int64 { o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) if o != 0 { @@ -159,20 +159,20 @@ func (rcv *RecordBatch) VariadicBufferCountsLength() int { return 0 } -/// Some types such as Utf8View are represented using a variable number of buffers. -/// For each such Field in the pre-ordered flattened logical schema, there will be -/// an entry in variadicBufferCounts to indicate the number of number of variadic -/// buffers which belong to that Field in the current RecordBatch. -/// -/// For example, the schema -/// col1: Struct -/// col2: Utf8View -/// contains two Fields with variadic buffers so variadicBufferCounts will have -/// two entries, the first counting the variadic buffers of `col1.beta` and the -/// second counting `col2`'s. -/// -/// This field may be omitted if and only if the schema contains no Fields with -/// a variable number of buffers, such as BinaryView and Utf8View. +// / Some types such as Utf8View are represented using a variable number of buffers. +// / For each such Field in the pre-ordered flattened logical schema, there will be +// / an entry in variadicBufferCounts to indicate the number of number of variadic +// / buffers which belong to that Field in the current RecordBatch. +// / +// / For example, the schema +// / col1: Struct +// / col2: Utf8View +// / contains two Fields with variadic buffers so variadicBufferCounts will have +// / two entries, the first counting the variadic buffers of `col1.beta` and the +// / second counting `col2`'s. +// / +// / This field may be omitted if and only if the schema contains no Fields with +// / a variable number of buffers, such as BinaryView and Utf8View. func (rcv *RecordBatch) MutateVariadicBufferCounts(j int, n int64) bool { o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) if o != 0 { diff --git a/go/arrow/internal/flatbuf/RunEndEncoded.go b/go/arrow/internal/flatbuf/RunEndEncoded.go index fa414c1bf0eed..b88460b2e22bc 100644 --- a/go/arrow/internal/flatbuf/RunEndEncoded.go +++ b/go/arrow/internal/flatbuf/RunEndEncoded.go @@ -22,11 +22,11 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// Contains two child arrays, run_ends and values. -/// The run_ends child array must be a 16/32/64-bit integer array -/// which encodes the indices at which the run with the value in -/// each corresponding index in the values child array ends. -/// Like list/struct types, the value array can be of any type. +// / Contains two child arrays, run_ends and values. +// / The run_ends child array must be a 16/32/64-bit integer array +// / which encodes the indices at which the run with the value in +// / each corresponding index in the values child array ends. +// / Like list/struct types, the value array can be of any type. type RunEndEncoded struct { _tab flatbuffers.Table } diff --git a/go/arrow/internal/flatbuf/Schema.go b/go/arrow/internal/flatbuf/Schema.go index 4ee5ecc9e5e40..ae5b248a766e3 100644 --- a/go/arrow/internal/flatbuf/Schema.go +++ b/go/arrow/internal/flatbuf/Schema.go @@ -22,8 +22,8 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// ---------------------------------------------------------------------- -/// A Schema describes the columns in a row batch +// / ---------------------------------------------------------------------- +// / A Schema describes the columns in a row batch type Schema struct { _tab flatbuffers.Table } @@ -44,9 +44,9 @@ func (rcv *Schema) Table() flatbuffers.Table { return rcv._tab } -/// endianness of the buffer -/// it is Little Endian by default -/// if endianness doesn't match the underlying system then the vectors need to be converted +// / endianness of the buffer +// / it is Little Endian by default +// / if endianness doesn't match the underlying system then the vectors need to be converted func (rcv *Schema) Endianness() Endianness { o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) if o != 0 { @@ -55,9 +55,9 @@ func (rcv *Schema) Endianness() Endianness { return 0 } -/// endianness of the buffer -/// it is Little Endian by default -/// if endianness doesn't match the underlying system then the vectors need to be converted +// / endianness of the buffer +// / it is Little Endian by default +// / if endianness doesn't match the underlying system then the vectors need to be converted func (rcv *Schema) MutateEndianness(n Endianness) bool { return rcv._tab.MutateInt16Slot(4, int16(n)) } @@ -102,7 +102,7 @@ func (rcv *Schema) CustomMetadataLength() int { return 0 } -/// Features used in the stream/file. +// / Features used in the stream/file. func (rcv *Schema) Features(j int) Feature { o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) if o != 0 { @@ -120,7 +120,7 @@ func (rcv *Schema) FeaturesLength() int { return 0 } -/// Features used in the stream/file. +// / Features used in the stream/file. func (rcv *Schema) MutateFeatures(j int, n Feature) bool { o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) if o != 0 { diff --git a/go/arrow/internal/flatbuf/SparseMatrixIndexCSR.go b/go/arrow/internal/flatbuf/SparseMatrixIndexCSR.go index de8217650b281..2477af100355c 100644 --- a/go/arrow/internal/flatbuf/SparseMatrixIndexCSR.go +++ b/go/arrow/internal/flatbuf/SparseMatrixIndexCSR.go @@ -22,7 +22,7 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// Compressed Sparse Row format, that is matrix-specific. +// / Compressed Sparse Row format, that is matrix-specific. type SparseMatrixIndexCSR struct { _tab flatbuffers.Table } @@ -43,7 +43,7 @@ func (rcv *SparseMatrixIndexCSR) Table() flatbuffers.Table { return rcv._tab } -/// The type of values in indptrBuffer +// / The type of values in indptrBuffer func (rcv *SparseMatrixIndexCSR) IndptrType(obj *Int) *Int { o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) if o != 0 { @@ -57,29 +57,29 @@ func (rcv *SparseMatrixIndexCSR) IndptrType(obj *Int) *Int { return nil } -/// The type of values in indptrBuffer -/// indptrBuffer stores the location and size of indptr array that -/// represents the range of the rows. -/// The i-th row spans from indptr[i] to indptr[i+1] in the data. -/// The length of this array is 1 + (the number of rows), and the type -/// of index value is long. -/// -/// For example, let X be the following 6x4 matrix: -/// -/// X := [[0, 1, 2, 0], -/// [0, 0, 3, 0], -/// [0, 4, 0, 5], -/// [0, 0, 0, 0], -/// [6, 0, 7, 8], -/// [0, 9, 0, 0]]. -/// -/// The array of non-zero values in X is: -/// -/// values(X) = [1, 2, 3, 4, 5, 6, 7, 8, 9]. -/// -/// And the indptr of X is: -/// -/// indptr(X) = [0, 2, 3, 5, 5, 8, 10]. +// / The type of values in indptrBuffer +// / indptrBuffer stores the location and size of indptr array that +// / represents the range of the rows. +// / The i-th row spans from indptr[i] to indptr[i+1] in the data. +// / The length of this array is 1 + (the number of rows), and the type +// / of index value is long. +// / +// / For example, let X be the following 6x4 matrix: +// / +// / X := [[0, 1, 2, 0], +// / [0, 0, 3, 0], +// / [0, 4, 0, 5], +// / [0, 0, 0, 0], +// / [6, 0, 7, 8], +// / [0, 9, 0, 0]]. +// / +// / The array of non-zero values in X is: +// / +// / values(X) = [1, 2, 3, 4, 5, 6, 7, 8, 9]. +// / +// / And the indptr of X is: +// / +// / indptr(X) = [0, 2, 3, 5, 5, 8, 10]. func (rcv *SparseMatrixIndexCSR) IndptrBuffer(obj *Buffer) *Buffer { o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) if o != 0 { @@ -93,29 +93,29 @@ func (rcv *SparseMatrixIndexCSR) IndptrBuffer(obj *Buffer) *Buffer { return nil } -/// indptrBuffer stores the location and size of indptr array that -/// represents the range of the rows. -/// The i-th row spans from indptr[i] to indptr[i+1] in the data. -/// The length of this array is 1 + (the number of rows), and the type -/// of index value is long. -/// -/// For example, let X be the following 6x4 matrix: -/// -/// X := [[0, 1, 2, 0], -/// [0, 0, 3, 0], -/// [0, 4, 0, 5], -/// [0, 0, 0, 0], -/// [6, 0, 7, 8], -/// [0, 9, 0, 0]]. -/// -/// The array of non-zero values in X is: -/// -/// values(X) = [1, 2, 3, 4, 5, 6, 7, 8, 9]. -/// -/// And the indptr of X is: -/// -/// indptr(X) = [0, 2, 3, 5, 5, 8, 10]. -/// The type of values in indicesBuffer +// / indptrBuffer stores the location and size of indptr array that +// / represents the range of the rows. +// / The i-th row spans from indptr[i] to indptr[i+1] in the data. +// / The length of this array is 1 + (the number of rows), and the type +// / of index value is long. +// / +// / For example, let X be the following 6x4 matrix: +// / +// / X := [[0, 1, 2, 0], +// / [0, 0, 3, 0], +// / [0, 4, 0, 5], +// / [0, 0, 0, 0], +// / [6, 0, 7, 8], +// / [0, 9, 0, 0]]. +// / +// / The array of non-zero values in X is: +// / +// / values(X) = [1, 2, 3, 4, 5, 6, 7, 8, 9]. +// / +// / And the indptr of X is: +// / +// / indptr(X) = [0, 2, 3, 5, 5, 8, 10]. +// / The type of values in indicesBuffer func (rcv *SparseMatrixIndexCSR) IndicesType(obj *Int) *Int { o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) if o != 0 { @@ -129,16 +129,16 @@ func (rcv *SparseMatrixIndexCSR) IndicesType(obj *Int) *Int { return nil } -/// The type of values in indicesBuffer -/// indicesBuffer stores the location and size of the array that -/// contains the column indices of the corresponding non-zero values. -/// The type of index value is long. -/// -/// For example, the indices of the above X is: -/// -/// indices(X) = [1, 2, 2, 1, 3, 0, 2, 3, 1]. -/// -/// Note that the indices are sorted in lexicographical order for each row. +// / The type of values in indicesBuffer +// / indicesBuffer stores the location and size of the array that +// / contains the column indices of the corresponding non-zero values. +// / The type of index value is long. +// / +// / For example, the indices of the above X is: +// / +// / indices(X) = [1, 2, 2, 1, 3, 0, 2, 3, 1]. +// / +// / Note that the indices are sorted in lexicographical order for each row. func (rcv *SparseMatrixIndexCSR) IndicesBuffer(obj *Buffer) *Buffer { o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) if o != 0 { @@ -152,15 +152,15 @@ func (rcv *SparseMatrixIndexCSR) IndicesBuffer(obj *Buffer) *Buffer { return nil } -/// indicesBuffer stores the location and size of the array that -/// contains the column indices of the corresponding non-zero values. -/// The type of index value is long. -/// -/// For example, the indices of the above X is: -/// -/// indices(X) = [1, 2, 2, 1, 3, 0, 2, 3, 1]. -/// -/// Note that the indices are sorted in lexicographical order for each row. +// / indicesBuffer stores the location and size of the array that +// / contains the column indices of the corresponding non-zero values. +// / The type of index value is long. +// / +// / For example, the indices of the above X is: +// / +// / indices(X) = [1, 2, 2, 1, 3, 0, 2, 3, 1]. +// / +// / Note that the indices are sorted in lexicographical order for each row. func SparseMatrixIndexCSRStart(builder *flatbuffers.Builder) { builder.StartObject(4) } diff --git a/go/arrow/internal/flatbuf/SparseMatrixIndexCSX.go b/go/arrow/internal/flatbuf/SparseMatrixIndexCSX.go index c28cc5d082fac..7f262deedbfc1 100644 --- a/go/arrow/internal/flatbuf/SparseMatrixIndexCSX.go +++ b/go/arrow/internal/flatbuf/SparseMatrixIndexCSX.go @@ -22,7 +22,7 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// Compressed Sparse format, that is matrix-specific. +// / Compressed Sparse format, that is matrix-specific. type SparseMatrixIndexCSX struct { _tab flatbuffers.Table } @@ -43,7 +43,7 @@ func (rcv *SparseMatrixIndexCSX) Table() flatbuffers.Table { return rcv._tab } -/// Which axis, row or column, is compressed +// / Which axis, row or column, is compressed func (rcv *SparseMatrixIndexCSX) CompressedAxis() SparseMatrixCompressedAxis { o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) if o != 0 { @@ -52,12 +52,12 @@ func (rcv *SparseMatrixIndexCSX) CompressedAxis() SparseMatrixCompressedAxis { return 0 } -/// Which axis, row or column, is compressed +// / Which axis, row or column, is compressed func (rcv *SparseMatrixIndexCSX) MutateCompressedAxis(n SparseMatrixCompressedAxis) bool { return rcv._tab.MutateInt16Slot(4, int16(n)) } -/// The type of values in indptrBuffer +// / The type of values in indptrBuffer func (rcv *SparseMatrixIndexCSX) IndptrType(obj *Int) *Int { o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) if o != 0 { @@ -71,30 +71,30 @@ func (rcv *SparseMatrixIndexCSX) IndptrType(obj *Int) *Int { return nil } -/// The type of values in indptrBuffer -/// indptrBuffer stores the location and size of indptr array that -/// represents the range of the rows. -/// The i-th row spans from `indptr[i]` to `indptr[i+1]` in the data. -/// The length of this array is 1 + (the number of rows), and the type -/// of index value is long. -/// -/// For example, let X be the following 6x4 matrix: -/// ```text -/// X := [[0, 1, 2, 0], -/// [0, 0, 3, 0], -/// [0, 4, 0, 5], -/// [0, 0, 0, 0], -/// [6, 0, 7, 8], -/// [0, 9, 0, 0]]. -/// ``` -/// The array of non-zero values in X is: -/// ```text -/// values(X) = [1, 2, 3, 4, 5, 6, 7, 8, 9]. -/// ``` -/// And the indptr of X is: -/// ```text -/// indptr(X) = [0, 2, 3, 5, 5, 8, 10]. -/// ``` +// / The type of values in indptrBuffer +// / indptrBuffer stores the location and size of indptr array that +// / represents the range of the rows. +// / The i-th row spans from `indptr[i]` to `indptr[i+1]` in the data. +// / The length of this array is 1 + (the number of rows), and the type +// / of index value is long. +// / +// / For example, let X be the following 6x4 matrix: +// / ```text +// / X := [[0, 1, 2, 0], +// / [0, 0, 3, 0], +// / [0, 4, 0, 5], +// / [0, 0, 0, 0], +// / [6, 0, 7, 8], +// / [0, 9, 0, 0]]. +// / ``` +// / The array of non-zero values in X is: +// / ```text +// / values(X) = [1, 2, 3, 4, 5, 6, 7, 8, 9]. +// / ``` +// / And the indptr of X is: +// / ```text +// / indptr(X) = [0, 2, 3, 5, 5, 8, 10]. +// / ``` func (rcv *SparseMatrixIndexCSX) IndptrBuffer(obj *Buffer) *Buffer { o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) if o != 0 { @@ -108,30 +108,30 @@ func (rcv *SparseMatrixIndexCSX) IndptrBuffer(obj *Buffer) *Buffer { return nil } -/// indptrBuffer stores the location and size of indptr array that -/// represents the range of the rows. -/// The i-th row spans from `indptr[i]` to `indptr[i+1]` in the data. -/// The length of this array is 1 + (the number of rows), and the type -/// of index value is long. -/// -/// For example, let X be the following 6x4 matrix: -/// ```text -/// X := [[0, 1, 2, 0], -/// [0, 0, 3, 0], -/// [0, 4, 0, 5], -/// [0, 0, 0, 0], -/// [6, 0, 7, 8], -/// [0, 9, 0, 0]]. -/// ``` -/// The array of non-zero values in X is: -/// ```text -/// values(X) = [1, 2, 3, 4, 5, 6, 7, 8, 9]. -/// ``` -/// And the indptr of X is: -/// ```text -/// indptr(X) = [0, 2, 3, 5, 5, 8, 10]. -/// ``` -/// The type of values in indicesBuffer +// / indptrBuffer stores the location and size of indptr array that +// / represents the range of the rows. +// / The i-th row spans from `indptr[i]` to `indptr[i+1]` in the data. +// / The length of this array is 1 + (the number of rows), and the type +// / of index value is long. +// / +// / For example, let X be the following 6x4 matrix: +// / ```text +// / X := [[0, 1, 2, 0], +// / [0, 0, 3, 0], +// / [0, 4, 0, 5], +// / [0, 0, 0, 0], +// / [6, 0, 7, 8], +// / [0, 9, 0, 0]]. +// / ``` +// / The array of non-zero values in X is: +// / ```text +// / values(X) = [1, 2, 3, 4, 5, 6, 7, 8, 9]. +// / ``` +// / And the indptr of X is: +// / ```text +// / indptr(X) = [0, 2, 3, 5, 5, 8, 10]. +// / ``` +// / The type of values in indicesBuffer func (rcv *SparseMatrixIndexCSX) IndicesType(obj *Int) *Int { o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) if o != 0 { @@ -145,16 +145,16 @@ func (rcv *SparseMatrixIndexCSX) IndicesType(obj *Int) *Int { return nil } -/// The type of values in indicesBuffer -/// indicesBuffer stores the location and size of the array that -/// contains the column indices of the corresponding non-zero values. -/// The type of index value is long. -/// -/// For example, the indices of the above X is: -/// ```text -/// indices(X) = [1, 2, 2, 1, 3, 0, 2, 3, 1]. -/// ``` -/// Note that the indices are sorted in lexicographical order for each row. +// / The type of values in indicesBuffer +// / indicesBuffer stores the location and size of the array that +// / contains the column indices of the corresponding non-zero values. +// / The type of index value is long. +// / +// / For example, the indices of the above X is: +// / ```text +// / indices(X) = [1, 2, 2, 1, 3, 0, 2, 3, 1]. +// / ``` +// / Note that the indices are sorted in lexicographical order for each row. func (rcv *SparseMatrixIndexCSX) IndicesBuffer(obj *Buffer) *Buffer { o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) if o != 0 { @@ -168,15 +168,15 @@ func (rcv *SparseMatrixIndexCSX) IndicesBuffer(obj *Buffer) *Buffer { return nil } -/// indicesBuffer stores the location and size of the array that -/// contains the column indices of the corresponding non-zero values. -/// The type of index value is long. -/// -/// For example, the indices of the above X is: -/// ```text -/// indices(X) = [1, 2, 2, 1, 3, 0, 2, 3, 1]. -/// ``` -/// Note that the indices are sorted in lexicographical order for each row. +// / indicesBuffer stores the location and size of the array that +// / contains the column indices of the corresponding non-zero values. +// / The type of index value is long. +// / +// / For example, the indices of the above X is: +// / ```text +// / indices(X) = [1, 2, 2, 1, 3, 0, 2, 3, 1]. +// / ``` +// / Note that the indices are sorted in lexicographical order for each row. func SparseMatrixIndexCSXStart(builder *flatbuffers.Builder) { builder.StartObject(5) } diff --git a/go/arrow/internal/flatbuf/SparseTensor.go b/go/arrow/internal/flatbuf/SparseTensor.go index 6f3f55797d755..8f67e1fc08b84 100644 --- a/go/arrow/internal/flatbuf/SparseTensor.go +++ b/go/arrow/internal/flatbuf/SparseTensor.go @@ -54,9 +54,9 @@ func (rcv *SparseTensor) MutateTypeType(n Type) bool { return rcv._tab.MutateByteSlot(4, byte(n)) } -/// The type of data contained in a value cell. -/// Currently only fixed-width value types are supported, -/// no strings or nested types. +// / The type of data contained in a value cell. +// / Currently only fixed-width value types are supported, +// / no strings or nested types. func (rcv *SparseTensor) Type(obj *flatbuffers.Table) bool { o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) if o != 0 { @@ -66,10 +66,10 @@ func (rcv *SparseTensor) Type(obj *flatbuffers.Table) bool { return false } -/// The type of data contained in a value cell. -/// Currently only fixed-width value types are supported, -/// no strings or nested types. -/// The dimensions of the tensor, optionally named. +// / The type of data contained in a value cell. +// / Currently only fixed-width value types are supported, +// / no strings or nested types. +// / The dimensions of the tensor, optionally named. func (rcv *SparseTensor) Shape(obj *TensorDim, j int) bool { o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) if o != 0 { @@ -90,8 +90,8 @@ func (rcv *SparseTensor) ShapeLength() int { return 0 } -/// The dimensions of the tensor, optionally named. -/// The number of non-zero values in a sparse tensor. +// / The dimensions of the tensor, optionally named. +// / The number of non-zero values in a sparse tensor. func (rcv *SparseTensor) NonZeroLength() int64 { o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) if o != 0 { @@ -100,7 +100,7 @@ func (rcv *SparseTensor) NonZeroLength() int64 { return 0 } -/// The number of non-zero values in a sparse tensor. +// / The number of non-zero values in a sparse tensor. func (rcv *SparseTensor) MutateNonZeroLength(n int64) bool { return rcv._tab.MutateInt64Slot(10, n) } @@ -117,7 +117,7 @@ func (rcv *SparseTensor) MutateSparseIndexType(n SparseTensorIndex) bool { return rcv._tab.MutateByteSlot(12, byte(n)) } -/// Sparse tensor index +// / Sparse tensor index func (rcv *SparseTensor) SparseIndex(obj *flatbuffers.Table) bool { o := flatbuffers.UOffsetT(rcv._tab.Offset(14)) if o != 0 { @@ -127,8 +127,8 @@ func (rcv *SparseTensor) SparseIndex(obj *flatbuffers.Table) bool { return false } -/// Sparse tensor index -/// The location and size of the tensor's data +// / Sparse tensor index +// / The location and size of the tensor's data func (rcv *SparseTensor) Data(obj *Buffer) *Buffer { o := flatbuffers.UOffsetT(rcv._tab.Offset(16)) if o != 0 { @@ -142,7 +142,7 @@ func (rcv *SparseTensor) Data(obj *Buffer) *Buffer { return nil } -/// The location and size of the tensor's data +// / The location and size of the tensor's data func SparseTensorStart(builder *flatbuffers.Builder) { builder.StartObject(7) } diff --git a/go/arrow/internal/flatbuf/SparseTensorIndexCOO.go b/go/arrow/internal/flatbuf/SparseTensorIndexCOO.go index f8eee99fa691e..bf1c218e2e415 100644 --- a/go/arrow/internal/flatbuf/SparseTensorIndexCOO.go +++ b/go/arrow/internal/flatbuf/SparseTensorIndexCOO.go @@ -22,38 +22,38 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// ---------------------------------------------------------------------- -/// EXPERIMENTAL: Data structures for sparse tensors -/// Coordinate (COO) format of sparse tensor index. -/// -/// COO's index list are represented as a NxM matrix, -/// where N is the number of non-zero values, -/// and M is the number of dimensions of a sparse tensor. -/// -/// indicesBuffer stores the location and size of the data of this indices -/// matrix. The value type and the stride of the indices matrix is -/// specified in indicesType and indicesStrides fields. -/// -/// For example, let X be a 2x3x4x5 tensor, and it has the following -/// 6 non-zero values: -/// ```text -/// X[0, 1, 2, 0] := 1 -/// X[1, 1, 2, 3] := 2 -/// X[0, 2, 1, 0] := 3 -/// X[0, 1, 3, 0] := 4 -/// X[0, 1, 2, 1] := 5 -/// X[1, 2, 0, 4] := 6 -/// ``` -/// In COO format, the index matrix of X is the following 4x6 matrix: -/// ```text -/// [[0, 0, 0, 0, 1, 1], -/// [1, 1, 1, 2, 1, 2], -/// [2, 2, 3, 1, 2, 0], -/// [0, 1, 0, 0, 3, 4]] -/// ``` -/// When isCanonical is true, the indices is sorted in lexicographical order -/// (row-major order), and it does not have duplicated entries. Otherwise, -/// the indices may not be sorted, or may have duplicated entries. +// / ---------------------------------------------------------------------- +// / EXPERIMENTAL: Data structures for sparse tensors +// / Coordinate (COO) format of sparse tensor index. +// / +// / COO's index list are represented as a NxM matrix, +// / where N is the number of non-zero values, +// / and M is the number of dimensions of a sparse tensor. +// / +// / indicesBuffer stores the location and size of the data of this indices +// / matrix. The value type and the stride of the indices matrix is +// / specified in indicesType and indicesStrides fields. +// / +// / For example, let X be a 2x3x4x5 tensor, and it has the following +// / 6 non-zero values: +// / ```text +// / X[0, 1, 2, 0] := 1 +// / X[1, 1, 2, 3] := 2 +// / X[0, 2, 1, 0] := 3 +// / X[0, 1, 3, 0] := 4 +// / X[0, 1, 2, 1] := 5 +// / X[1, 2, 0, 4] := 6 +// / ``` +// / In COO format, the index matrix of X is the following 4x6 matrix: +// / ```text +// / [[0, 0, 0, 0, 1, 1], +// / [1, 1, 1, 2, 1, 2], +// / [2, 2, 3, 1, 2, 0], +// / [0, 1, 0, 0, 3, 4]] +// / ``` +// / When isCanonical is true, the indices is sorted in lexicographical order +// / (row-major order), and it does not have duplicated entries. Otherwise, +// / the indices may not be sorted, or may have duplicated entries. type SparseTensorIndexCOO struct { _tab flatbuffers.Table } @@ -74,7 +74,7 @@ func (rcv *SparseTensorIndexCOO) Table() flatbuffers.Table { return rcv._tab } -/// The type of values in indicesBuffer +// / The type of values in indicesBuffer func (rcv *SparseTensorIndexCOO) IndicesType(obj *Int) *Int { o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) if o != 0 { @@ -88,9 +88,9 @@ func (rcv *SparseTensorIndexCOO) IndicesType(obj *Int) *Int { return nil } -/// The type of values in indicesBuffer -/// Non-negative byte offsets to advance one value cell along each dimension -/// If omitted, default to row-major order (C-like). +// / The type of values in indicesBuffer +// / Non-negative byte offsets to advance one value cell along each dimension +// / If omitted, default to row-major order (C-like). func (rcv *SparseTensorIndexCOO) IndicesStrides(j int) int64 { o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) if o != 0 { @@ -108,8 +108,8 @@ func (rcv *SparseTensorIndexCOO) IndicesStridesLength() int { return 0 } -/// Non-negative byte offsets to advance one value cell along each dimension -/// If omitted, default to row-major order (C-like). +// / Non-negative byte offsets to advance one value cell along each dimension +// / If omitted, default to row-major order (C-like). func (rcv *SparseTensorIndexCOO) MutateIndicesStrides(j int, n int64) bool { o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) if o != 0 { @@ -119,7 +119,7 @@ func (rcv *SparseTensorIndexCOO) MutateIndicesStrides(j int, n int64) bool { return false } -/// The location and size of the indices matrix's data +// / The location and size of the indices matrix's data func (rcv *SparseTensorIndexCOO) IndicesBuffer(obj *Buffer) *Buffer { o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) if o != 0 { @@ -133,12 +133,12 @@ func (rcv *SparseTensorIndexCOO) IndicesBuffer(obj *Buffer) *Buffer { return nil } -/// The location and size of the indices matrix's data -/// This flag is true if and only if the indices matrix is sorted in -/// row-major order, and does not have duplicated entries. -/// This sort order is the same as of Tensorflow's SparseTensor, -/// but it is inverse order of SciPy's canonical coo_matrix -/// (SciPy employs column-major order for its coo_matrix). +// / The location and size of the indices matrix's data +// / This flag is true if and only if the indices matrix is sorted in +// / row-major order, and does not have duplicated entries. +// / This sort order is the same as of Tensorflow's SparseTensor, +// / but it is inverse order of SciPy's canonical coo_matrix +// / (SciPy employs column-major order for its coo_matrix). func (rcv *SparseTensorIndexCOO) IsCanonical() bool { o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) if o != 0 { @@ -147,11 +147,11 @@ func (rcv *SparseTensorIndexCOO) IsCanonical() bool { return false } -/// This flag is true if and only if the indices matrix is sorted in -/// row-major order, and does not have duplicated entries. -/// This sort order is the same as of Tensorflow's SparseTensor, -/// but it is inverse order of SciPy's canonical coo_matrix -/// (SciPy employs column-major order for its coo_matrix). +// / This flag is true if and only if the indices matrix is sorted in +// / row-major order, and does not have duplicated entries. +// / This sort order is the same as of Tensorflow's SparseTensor, +// / but it is inverse order of SciPy's canonical coo_matrix +// / (SciPy employs column-major order for its coo_matrix). func (rcv *SparseTensorIndexCOO) MutateIsCanonical(n bool) bool { return rcv._tab.MutateBoolSlot(10, n) } diff --git a/go/arrow/internal/flatbuf/SparseTensorIndexCSF.go b/go/arrow/internal/flatbuf/SparseTensorIndexCSF.go index a824c84ebfe2e..66226e0412c21 100644 --- a/go/arrow/internal/flatbuf/SparseTensorIndexCSF.go +++ b/go/arrow/internal/flatbuf/SparseTensorIndexCSF.go @@ -22,7 +22,7 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// Compressed Sparse Fiber (CSF) sparse tensor index. +// / Compressed Sparse Fiber (CSF) sparse tensor index. type SparseTensorIndexCSF struct { _tab flatbuffers.Table } @@ -43,37 +43,37 @@ func (rcv *SparseTensorIndexCSF) Table() flatbuffers.Table { return rcv._tab } -/// CSF is a generalization of compressed sparse row (CSR) index. -/// See [smith2017knl](http://shaden.io/pub-files/smith2017knl.pdf) -/// -/// CSF index recursively compresses each dimension of a tensor into a set -/// of prefix trees. Each path from a root to leaf forms one tensor -/// non-zero index. CSF is implemented with two arrays of buffers and one -/// arrays of integers. -/// -/// For example, let X be a 2x3x4x5 tensor and let it have the following -/// 8 non-zero values: -/// ```text -/// X[0, 0, 0, 1] := 1 -/// X[0, 0, 0, 2] := 2 -/// X[0, 1, 0, 0] := 3 -/// X[0, 1, 0, 2] := 4 -/// X[0, 1, 1, 0] := 5 -/// X[1, 1, 1, 0] := 6 -/// X[1, 1, 1, 1] := 7 -/// X[1, 1, 1, 2] := 8 -/// ``` -/// As a prefix tree this would be represented as: -/// ```text -/// 0 1 -/// / \ | -/// 0 1 1 -/// / / \ | -/// 0 0 1 1 -/// /| /| | /| | -/// 1 2 0 2 0 0 1 2 -/// ``` -/// The type of values in indptrBuffers +// / CSF is a generalization of compressed sparse row (CSR) index. +// / See [smith2017knl](http://shaden.io/pub-files/smith2017knl.pdf) +// / +// / CSF index recursively compresses each dimension of a tensor into a set +// / of prefix trees. Each path from a root to leaf forms one tensor +// / non-zero index. CSF is implemented with two arrays of buffers and one +// / arrays of integers. +// / +// / For example, let X be a 2x3x4x5 tensor and let it have the following +// / 8 non-zero values: +// / ```text +// / X[0, 0, 0, 1] := 1 +// / X[0, 0, 0, 2] := 2 +// / X[0, 1, 0, 0] := 3 +// / X[0, 1, 0, 2] := 4 +// / X[0, 1, 1, 0] := 5 +// / X[1, 1, 1, 0] := 6 +// / X[1, 1, 1, 1] := 7 +// / X[1, 1, 1, 2] := 8 +// / ``` +// / As a prefix tree this would be represented as: +// / ```text +// / 0 1 +// / / \ | +// / 0 1 1 +// / / / \ | +// / 0 0 1 1 +// / /| /| | /| | +// / 1 2 0 2 0 0 1 2 +// / ``` +// / The type of values in indptrBuffers func (rcv *SparseTensorIndexCSF) IndptrType(obj *Int) *Int { o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) if o != 0 { @@ -87,51 +87,51 @@ func (rcv *SparseTensorIndexCSF) IndptrType(obj *Int) *Int { return nil } -/// CSF is a generalization of compressed sparse row (CSR) index. -/// See [smith2017knl](http://shaden.io/pub-files/smith2017knl.pdf) -/// -/// CSF index recursively compresses each dimension of a tensor into a set -/// of prefix trees. Each path from a root to leaf forms one tensor -/// non-zero index. CSF is implemented with two arrays of buffers and one -/// arrays of integers. -/// -/// For example, let X be a 2x3x4x5 tensor and let it have the following -/// 8 non-zero values: -/// ```text -/// X[0, 0, 0, 1] := 1 -/// X[0, 0, 0, 2] := 2 -/// X[0, 1, 0, 0] := 3 -/// X[0, 1, 0, 2] := 4 -/// X[0, 1, 1, 0] := 5 -/// X[1, 1, 1, 0] := 6 -/// X[1, 1, 1, 1] := 7 -/// X[1, 1, 1, 2] := 8 -/// ``` -/// As a prefix tree this would be represented as: -/// ```text -/// 0 1 -/// / \ | -/// 0 1 1 -/// / / \ | -/// 0 0 1 1 -/// /| /| | /| | -/// 1 2 0 2 0 0 1 2 -/// ``` -/// The type of values in indptrBuffers -/// indptrBuffers stores the sparsity structure. -/// Each two consecutive dimensions in a tensor correspond to a buffer in -/// indptrBuffers. A pair of consecutive values at `indptrBuffers[dim][i]` -/// and `indptrBuffers[dim][i + 1]` signify a range of nodes in -/// `indicesBuffers[dim + 1]` who are children of `indicesBuffers[dim][i]` node. -/// -/// For example, the indptrBuffers for the above X is: -/// ```text -/// indptrBuffer(X) = [ -/// [0, 2, 3], -/// [0, 1, 3, 4], -/// [0, 2, 4, 5, 8] -/// ]. -/// ``` +// / CSF is a generalization of compressed sparse row (CSR) index. +// / See [smith2017knl](http://shaden.io/pub-files/smith2017knl.pdf) +// / +// / CSF index recursively compresses each dimension of a tensor into a set +// / of prefix trees. Each path from a root to leaf forms one tensor +// / non-zero index. CSF is implemented with two arrays of buffers and one +// / arrays of integers. +// / +// / For example, let X be a 2x3x4x5 tensor and let it have the following +// / 8 non-zero values: +// / ```text +// / X[0, 0, 0, 1] := 1 +// / X[0, 0, 0, 2] := 2 +// / X[0, 1, 0, 0] := 3 +// / X[0, 1, 0, 2] := 4 +// / X[0, 1, 1, 0] := 5 +// / X[1, 1, 1, 0] := 6 +// / X[1, 1, 1, 1] := 7 +// / X[1, 1, 1, 2] := 8 +// / ``` +// / As a prefix tree this would be represented as: +// / ```text +// / 0 1 +// / / \ | +// / 0 1 1 +// / / / \ | +// / 0 0 1 1 +// / /| /| | /| | +// / 1 2 0 2 0 0 1 2 +// / ``` +// / The type of values in indptrBuffers +// / indptrBuffers stores the sparsity structure. +// / Each two consecutive dimensions in a tensor correspond to a buffer in +// / indptrBuffers. A pair of consecutive values at `indptrBuffers[dim][i]` +// / and `indptrBuffers[dim][i + 1]` signify a range of nodes in +// / `indicesBuffers[dim + 1]` who are children of `indicesBuffers[dim][i]` node. +// / +// / For example, the indptrBuffers for the above X is: +// / ```text +// / indptrBuffer(X) = [ +// / [0, 2, 3], +// / [0, 1, 3, 4], +// / [0, 2, 4, 5, 8] +// / ]. +// / ``` func (rcv *SparseTensorIndexCSF) IndptrBuffers(obj *Buffer, j int) bool { o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) if o != 0 { @@ -151,21 +151,21 @@ func (rcv *SparseTensorIndexCSF) IndptrBuffersLength() int { return 0 } -/// indptrBuffers stores the sparsity structure. -/// Each two consecutive dimensions in a tensor correspond to a buffer in -/// indptrBuffers. A pair of consecutive values at `indptrBuffers[dim][i]` -/// and `indptrBuffers[dim][i + 1]` signify a range of nodes in -/// `indicesBuffers[dim + 1]` who are children of `indicesBuffers[dim][i]` node. -/// -/// For example, the indptrBuffers for the above X is: -/// ```text -/// indptrBuffer(X) = [ -/// [0, 2, 3], -/// [0, 1, 3, 4], -/// [0, 2, 4, 5, 8] -/// ]. -/// ``` -/// The type of values in indicesBuffers +// / indptrBuffers stores the sparsity structure. +// / Each two consecutive dimensions in a tensor correspond to a buffer in +// / indptrBuffers. A pair of consecutive values at `indptrBuffers[dim][i]` +// / and `indptrBuffers[dim][i + 1]` signify a range of nodes in +// / `indicesBuffers[dim + 1]` who are children of `indicesBuffers[dim][i]` node. +// / +// / For example, the indptrBuffers for the above X is: +// / ```text +// / indptrBuffer(X) = [ +// / [0, 2, 3], +// / [0, 1, 3, 4], +// / [0, 2, 4, 5, 8] +// / ]. +// / ``` +// / The type of values in indicesBuffers func (rcv *SparseTensorIndexCSF) IndicesType(obj *Int) *Int { o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) if o != 0 { @@ -179,18 +179,18 @@ func (rcv *SparseTensorIndexCSF) IndicesType(obj *Int) *Int { return nil } -/// The type of values in indicesBuffers -/// indicesBuffers stores values of nodes. -/// Each tensor dimension corresponds to a buffer in indicesBuffers. -/// For example, the indicesBuffers for the above X is: -/// ```text -/// indicesBuffer(X) = [ -/// [0, 1], -/// [0, 1, 1], -/// [0, 0, 1, 1], -/// [1, 2, 0, 2, 0, 0, 1, 2] -/// ]. -/// ``` +// / The type of values in indicesBuffers +// / indicesBuffers stores values of nodes. +// / Each tensor dimension corresponds to a buffer in indicesBuffers. +// / For example, the indicesBuffers for the above X is: +// / ```text +// / indicesBuffer(X) = [ +// / [0, 1], +// / [0, 1, 1], +// / [0, 0, 1, 1], +// / [1, 2, 0, 2, 0, 0, 1, 2] +// / ]. +// / ``` func (rcv *SparseTensorIndexCSF) IndicesBuffers(obj *Buffer, j int) bool { o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) if o != 0 { @@ -210,23 +210,23 @@ func (rcv *SparseTensorIndexCSF) IndicesBuffersLength() int { return 0 } -/// indicesBuffers stores values of nodes. -/// Each tensor dimension corresponds to a buffer in indicesBuffers. -/// For example, the indicesBuffers for the above X is: -/// ```text -/// indicesBuffer(X) = [ -/// [0, 1], -/// [0, 1, 1], -/// [0, 0, 1, 1], -/// [1, 2, 0, 2, 0, 0, 1, 2] -/// ]. -/// ``` -/// axisOrder stores the sequence in which dimensions were traversed to -/// produce the prefix tree. -/// For example, the axisOrder for the above X is: -/// ```text -/// axisOrder(X) = [0, 1, 2, 3]. -/// ``` +// / indicesBuffers stores values of nodes. +// / Each tensor dimension corresponds to a buffer in indicesBuffers. +// / For example, the indicesBuffers for the above X is: +// / ```text +// / indicesBuffer(X) = [ +// / [0, 1], +// / [0, 1, 1], +// / [0, 0, 1, 1], +// / [1, 2, 0, 2, 0, 0, 1, 2] +// / ]. +// / ``` +// / axisOrder stores the sequence in which dimensions were traversed to +// / produce the prefix tree. +// / For example, the axisOrder for the above X is: +// / ```text +// / axisOrder(X) = [0, 1, 2, 3]. +// / ``` func (rcv *SparseTensorIndexCSF) AxisOrder(j int) int32 { o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) if o != 0 { @@ -244,12 +244,12 @@ func (rcv *SparseTensorIndexCSF) AxisOrderLength() int { return 0 } -/// axisOrder stores the sequence in which dimensions were traversed to -/// produce the prefix tree. -/// For example, the axisOrder for the above X is: -/// ```text -/// axisOrder(X) = [0, 1, 2, 3]. -/// ``` +// / axisOrder stores the sequence in which dimensions were traversed to +// / produce the prefix tree. +// / For example, the axisOrder for the above X is: +// / ```text +// / axisOrder(X) = [0, 1, 2, 3]. +// / ``` func (rcv *SparseTensorIndexCSF) MutateAxisOrder(j int, n int32) bool { o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) if o != 0 { diff --git a/go/arrow/internal/flatbuf/Struct_.go b/go/arrow/internal/flatbuf/Struct_.go index 427e7060382af..73752a17e00fa 100644 --- a/go/arrow/internal/flatbuf/Struct_.go +++ b/go/arrow/internal/flatbuf/Struct_.go @@ -22,9 +22,9 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// A Struct_ in the flatbuffer metadata is the same as an Arrow Struct -/// (according to the physical memory layout). We used Struct_ here as -/// Struct is a reserved word in Flatbuffers +// / A Struct_ in the flatbuffer metadata is the same as an Arrow Struct +// / (according to the physical memory layout). We used Struct_ here as +// / Struct is a reserved word in Flatbuffers type Struct_ struct { _tab flatbuffers.Table } diff --git a/go/arrow/internal/flatbuf/Tensor.go b/go/arrow/internal/flatbuf/Tensor.go index 39d70e351e3d6..47bfe8067b57b 100644 --- a/go/arrow/internal/flatbuf/Tensor.go +++ b/go/arrow/internal/flatbuf/Tensor.go @@ -54,8 +54,8 @@ func (rcv *Tensor) MutateTypeType(n Type) bool { return rcv._tab.MutateByteSlot(4, byte(n)) } -/// The type of data contained in a value cell. Currently only fixed-width -/// value types are supported, no strings or nested types +// / The type of data contained in a value cell. Currently only fixed-width +// / value types are supported, no strings or nested types func (rcv *Tensor) Type(obj *flatbuffers.Table) bool { o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) if o != 0 { @@ -65,9 +65,9 @@ func (rcv *Tensor) Type(obj *flatbuffers.Table) bool { return false } -/// The type of data contained in a value cell. Currently only fixed-width -/// value types are supported, no strings or nested types -/// The dimensions of the tensor, optionally named +// / The type of data contained in a value cell. Currently only fixed-width +// / value types are supported, no strings or nested types +// / The dimensions of the tensor, optionally named func (rcv *Tensor) Shape(obj *TensorDim, j int) bool { o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) if o != 0 { @@ -88,9 +88,9 @@ func (rcv *Tensor) ShapeLength() int { return 0 } -/// The dimensions of the tensor, optionally named -/// Non-negative byte offsets to advance one value cell along each dimension -/// If omitted, default to row-major order (C-like). +// / The dimensions of the tensor, optionally named +// / Non-negative byte offsets to advance one value cell along each dimension +// / If omitted, default to row-major order (C-like). func (rcv *Tensor) Strides(j int) int64 { o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) if o != 0 { @@ -108,8 +108,8 @@ func (rcv *Tensor) StridesLength() int { return 0 } -/// Non-negative byte offsets to advance one value cell along each dimension -/// If omitted, default to row-major order (C-like). +// / Non-negative byte offsets to advance one value cell along each dimension +// / If omitted, default to row-major order (C-like). func (rcv *Tensor) MutateStrides(j int, n int64) bool { o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) if o != 0 { @@ -119,7 +119,7 @@ func (rcv *Tensor) MutateStrides(j int, n int64) bool { return false } -/// The location and size of the tensor's data +// / The location and size of the tensor's data func (rcv *Tensor) Data(obj *Buffer) *Buffer { o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) if o != 0 { @@ -133,7 +133,7 @@ func (rcv *Tensor) Data(obj *Buffer) *Buffer { return nil } -/// The location and size of the tensor's data +// / The location and size of the tensor's data func TensorStart(builder *flatbuffers.Builder) { builder.StartObject(5) } diff --git a/go/arrow/internal/flatbuf/TensorDim.go b/go/arrow/internal/flatbuf/TensorDim.go index 14b82120887e9..c6413b6a8c0bd 100644 --- a/go/arrow/internal/flatbuf/TensorDim.go +++ b/go/arrow/internal/flatbuf/TensorDim.go @@ -22,9 +22,9 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// ---------------------------------------------------------------------- -/// Data structures for dense tensors -/// Shape data for a single axis in a tensor +// / ---------------------------------------------------------------------- +// / Data structures for dense tensors +// / Shape data for a single axis in a tensor type TensorDim struct { _tab flatbuffers.Table } @@ -45,7 +45,7 @@ func (rcv *TensorDim) Table() flatbuffers.Table { return rcv._tab } -/// Length of dimension +// / Length of dimension func (rcv *TensorDim) Size() int64 { o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) if o != 0 { @@ -54,12 +54,12 @@ func (rcv *TensorDim) Size() int64 { return 0 } -/// Length of dimension +// / Length of dimension func (rcv *TensorDim) MutateSize(n int64) bool { return rcv._tab.MutateInt64Slot(4, n) } -/// Name of the dimension, optional +// / Name of the dimension, optional func (rcv *TensorDim) Name() []byte { o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) if o != 0 { @@ -68,7 +68,7 @@ func (rcv *TensorDim) Name() []byte { return nil } -/// Name of the dimension, optional +// / Name of the dimension, optional func TensorDimStart(builder *flatbuffers.Builder) { builder.StartObject(2) } diff --git a/go/arrow/internal/flatbuf/Time.go b/go/arrow/internal/flatbuf/Time.go index 2fb6e4c110e0a..13038a6e33280 100644 --- a/go/arrow/internal/flatbuf/Time.go +++ b/go/arrow/internal/flatbuf/Time.go @@ -22,20 +22,20 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// Time is either a 32-bit or 64-bit signed integer type representing an -/// elapsed time since midnight, stored in either of four units: seconds, -/// milliseconds, microseconds or nanoseconds. -/// -/// The integer `bitWidth` depends on the `unit` and must be one of the following: -/// * SECOND and MILLISECOND: 32 bits -/// * MICROSECOND and NANOSECOND: 64 bits -/// -/// The allowed values are between 0 (inclusive) and 86400 (=24*60*60) seconds -/// (exclusive), adjusted for the time unit (for example, up to 86400000 -/// exclusive for the MILLISECOND unit). -/// This definition doesn't allow for leap seconds. Time values from -/// measurements with leap seconds will need to be corrected when ingesting -/// into Arrow (for example by replacing the value 86400 with 86399). +// / Time is either a 32-bit or 64-bit signed integer type representing an +// / elapsed time since midnight, stored in either of four units: seconds, +// / milliseconds, microseconds or nanoseconds. +// / +// / The integer `bitWidth` depends on the `unit` and must be one of the following: +// / * SECOND and MILLISECOND: 32 bits +// / * MICROSECOND and NANOSECOND: 64 bits +// / +// / The allowed values are between 0 (inclusive) and 86400 (=24*60*60) seconds +// / (exclusive), adjusted for the time unit (for example, up to 86400000 +// / exclusive for the MILLISECOND unit). +// / This definition doesn't allow for leap seconds. Time values from +// / measurements with leap seconds will need to be corrected when ingesting +// / into Arrow (for example by replacing the value 86400 with 86399). type Time struct { _tab flatbuffers.Table } diff --git a/go/arrow/internal/flatbuf/Timestamp.go b/go/arrow/internal/flatbuf/Timestamp.go index d0058e13e6545..ce172bacdd3c3 100644 --- a/go/arrow/internal/flatbuf/Timestamp.go +++ b/go/arrow/internal/flatbuf/Timestamp.go @@ -22,111 +22,111 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// Timestamp is a 64-bit signed integer representing an elapsed time since a -/// fixed epoch, stored in either of four units: seconds, milliseconds, -/// microseconds or nanoseconds, and is optionally annotated with a timezone. -/// -/// Timestamp values do not include any leap seconds (in other words, all -/// days are considered 86400 seconds long). -/// -/// Timestamps with a non-empty timezone -/// ------------------------------------ -/// -/// If a Timestamp column has a non-empty timezone value, its epoch is -/// 1970-01-01 00:00:00 (January 1st 1970, midnight) in the *UTC* timezone -/// (the Unix epoch), regardless of the Timestamp's own timezone. -/// -/// Therefore, timestamp values with a non-empty timezone correspond to -/// physical points in time together with some additional information about -/// how the data was obtained and/or how to display it (the timezone). -/// -/// For example, the timestamp value 0 with the timezone string "Europe/Paris" -/// corresponds to "January 1st 1970, 00h00" in the UTC timezone, but the -/// application may prefer to display it as "January 1st 1970, 01h00" in -/// the Europe/Paris timezone (which is the same physical point in time). -/// -/// One consequence is that timestamp values with a non-empty timezone -/// can be compared and ordered directly, since they all share the same -/// well-known point of reference (the Unix epoch). -/// -/// Timestamps with an unset / empty timezone -/// ----------------------------------------- -/// -/// If a Timestamp column has no timezone value, its epoch is -/// 1970-01-01 00:00:00 (January 1st 1970, midnight) in an *unknown* timezone. -/// -/// Therefore, timestamp values without a timezone cannot be meaningfully -/// interpreted as physical points in time, but only as calendar / clock -/// indications ("wall clock time") in an unspecified timezone. -/// -/// For example, the timestamp value 0 with an empty timezone string -/// corresponds to "January 1st 1970, 00h00" in an unknown timezone: there -/// is not enough information to interpret it as a well-defined physical -/// point in time. -/// -/// One consequence is that timestamp values without a timezone cannot -/// be reliably compared or ordered, since they may have different points of -/// reference. In particular, it is *not* possible to interpret an unset -/// or empty timezone as the same as "UTC". -/// -/// Conversion between timezones -/// ---------------------------- -/// -/// If a Timestamp column has a non-empty timezone, changing the timezone -/// to a different non-empty value is a metadata-only operation: -/// the timestamp values need not change as their point of reference remains -/// the same (the Unix epoch). -/// -/// However, if a Timestamp column has no timezone value, changing it to a -/// non-empty value requires to think about the desired semantics. -/// One possibility is to assume that the original timestamp values are -/// relative to the epoch of the timezone being set; timestamp values should -/// then adjusted to the Unix epoch (for example, changing the timezone from -/// empty to "Europe/Paris" would require converting the timestamp values -/// from "Europe/Paris" to "UTC", which seems counter-intuitive but is -/// nevertheless correct). -/// -/// Guidelines for encoding data from external libraries -/// ---------------------------------------------------- -/// -/// Date & time libraries often have multiple different data types for temporal -/// data. In order to ease interoperability between different implementations the -/// Arrow project has some recommendations for encoding these types into a Timestamp -/// column. -/// -/// An "instant" represents a physical point in time that has no relevant timezone -/// (for example, astronomical data). To encode an instant, use a Timestamp with -/// the timezone string set to "UTC", and make sure the Timestamp values -/// are relative to the UTC epoch (January 1st 1970, midnight). -/// -/// A "zoned date-time" represents a physical point in time annotated with an -/// informative timezone (for example, the timezone in which the data was -/// recorded). To encode a zoned date-time, use a Timestamp with the timezone -/// string set to the name of the timezone, and make sure the Timestamp values -/// are relative to the UTC epoch (January 1st 1970, midnight). -/// -/// (There is some ambiguity between an instant and a zoned date-time with the -/// UTC timezone. Both of these are stored the same in Arrow. Typically, -/// this distinction does not matter. If it does, then an application should -/// use custom metadata or an extension type to distinguish between the two cases.) -/// -/// An "offset date-time" represents a physical point in time combined with an -/// explicit offset from UTC. To encode an offset date-time, use a Timestamp -/// with the timezone string set to the numeric timezone offset string -/// (e.g. "+03:00"), and make sure the Timestamp values are relative to -/// the UTC epoch (January 1st 1970, midnight). -/// -/// A "naive date-time" (also called "local date-time" in some libraries) -/// represents a wall clock time combined with a calendar date, but with -/// no indication of how to map this information to a physical point in time. -/// Naive date-times must be handled with care because of this missing -/// information, and also because daylight saving time (DST) may make -/// some values ambiguous or nonexistent. A naive date-time may be -/// stored as a struct with Date and Time fields. However, it may also be -/// encoded into a Timestamp column with an empty timezone. The timestamp -/// values should be computed "as if" the timezone of the date-time values -/// was UTC; for example, the naive date-time "January 1st 1970, 00h00" would -/// be encoded as timestamp value 0. +// / Timestamp is a 64-bit signed integer representing an elapsed time since a +// / fixed epoch, stored in either of four units: seconds, milliseconds, +// / microseconds or nanoseconds, and is optionally annotated with a timezone. +// / +// / Timestamp values do not include any leap seconds (in other words, all +// / days are considered 86400 seconds long). +// / +// / Timestamps with a non-empty timezone +// / ------------------------------------ +// / +// / If a Timestamp column has a non-empty timezone value, its epoch is +// / 1970-01-01 00:00:00 (January 1st 1970, midnight) in the *UTC* timezone +// / (the Unix epoch), regardless of the Timestamp's own timezone. +// / +// / Therefore, timestamp values with a non-empty timezone correspond to +// / physical points in time together with some additional information about +// / how the data was obtained and/or how to display it (the timezone). +// / +// / For example, the timestamp value 0 with the timezone string "Europe/Paris" +// / corresponds to "January 1st 1970, 00h00" in the UTC timezone, but the +// / application may prefer to display it as "January 1st 1970, 01h00" in +// / the Europe/Paris timezone (which is the same physical point in time). +// / +// / One consequence is that timestamp values with a non-empty timezone +// / can be compared and ordered directly, since they all share the same +// / well-known point of reference (the Unix epoch). +// / +// / Timestamps with an unset / empty timezone +// / ----------------------------------------- +// / +// / If a Timestamp column has no timezone value, its epoch is +// / 1970-01-01 00:00:00 (January 1st 1970, midnight) in an *unknown* timezone. +// / +// / Therefore, timestamp values without a timezone cannot be meaningfully +// / interpreted as physical points in time, but only as calendar / clock +// / indications ("wall clock time") in an unspecified timezone. +// / +// / For example, the timestamp value 0 with an empty timezone string +// / corresponds to "January 1st 1970, 00h00" in an unknown timezone: there +// / is not enough information to interpret it as a well-defined physical +// / point in time. +// / +// / One consequence is that timestamp values without a timezone cannot +// / be reliably compared or ordered, since they may have different points of +// / reference. In particular, it is *not* possible to interpret an unset +// / or empty timezone as the same as "UTC". +// / +// / Conversion between timezones +// / ---------------------------- +// / +// / If a Timestamp column has a non-empty timezone, changing the timezone +// / to a different non-empty value is a metadata-only operation: +// / the timestamp values need not change as their point of reference remains +// / the same (the Unix epoch). +// / +// / However, if a Timestamp column has no timezone value, changing it to a +// / non-empty value requires to think about the desired semantics. +// / One possibility is to assume that the original timestamp values are +// / relative to the epoch of the timezone being set; timestamp values should +// / then adjusted to the Unix epoch (for example, changing the timezone from +// / empty to "Europe/Paris" would require converting the timestamp values +// / from "Europe/Paris" to "UTC", which seems counter-intuitive but is +// / nevertheless correct). +// / +// / Guidelines for encoding data from external libraries +// / ---------------------------------------------------- +// / +// / Date & time libraries often have multiple different data types for temporal +// / data. In order to ease interoperability between different implementations the +// / Arrow project has some recommendations for encoding these types into a Timestamp +// / column. +// / +// / An "instant" represents a physical point in time that has no relevant timezone +// / (for example, astronomical data). To encode an instant, use a Timestamp with +// / the timezone string set to "UTC", and make sure the Timestamp values +// / are relative to the UTC epoch (January 1st 1970, midnight). +// / +// / A "zoned date-time" represents a physical point in time annotated with an +// / informative timezone (for example, the timezone in which the data was +// / recorded). To encode a zoned date-time, use a Timestamp with the timezone +// / string set to the name of the timezone, and make sure the Timestamp values +// / are relative to the UTC epoch (January 1st 1970, midnight). +// / +// / (There is some ambiguity between an instant and a zoned date-time with the +// / UTC timezone. Both of these are stored the same in Arrow. Typically, +// / this distinction does not matter. If it does, then an application should +// / use custom metadata or an extension type to distinguish between the two cases.) +// / +// / An "offset date-time" represents a physical point in time combined with an +// / explicit offset from UTC. To encode an offset date-time, use a Timestamp +// / with the timezone string set to the numeric timezone offset string +// / (e.g. "+03:00"), and make sure the Timestamp values are relative to +// / the UTC epoch (January 1st 1970, midnight). +// / +// / A "naive date-time" (also called "local date-time" in some libraries) +// / represents a wall clock time combined with a calendar date, but with +// / no indication of how to map this information to a physical point in time. +// / Naive date-times must be handled with care because of this missing +// / information, and also because daylight saving time (DST) may make +// / some values ambiguous or nonexistent. A naive date-time may be +// / stored as a struct with Date and Time fields. However, it may also be +// / encoded into a Timestamp column with an empty timezone. The timestamp +// / values should be computed "as if" the timezone of the date-time values +// / was UTC; for example, the naive date-time "January 1st 1970, 00h00" would +// / be encoded as timestamp value 0. type Timestamp struct { _tab flatbuffers.Table } @@ -159,16 +159,16 @@ func (rcv *Timestamp) MutateUnit(n TimeUnit) bool { return rcv._tab.MutateInt16Slot(4, int16(n)) } -/// The timezone is an optional string indicating the name of a timezone, -/// one of: -/// -/// * As used in the Olson timezone database (the "tz database" or -/// "tzdata"), such as "America/New_York". -/// * An absolute timezone offset of the form "+XX:XX" or "-XX:XX", -/// such as "+07:30". -/// -/// Whether a timezone string is present indicates different semantics about -/// the data (see above). +// / The timezone is an optional string indicating the name of a timezone, +// / one of: +// / +// / * As used in the Olson timezone database (the "tz database" or +// / "tzdata"), such as "America/New_York". +// / * An absolute timezone offset of the form "+XX:XX" or "-XX:XX", +// / such as "+07:30". +// / +// / Whether a timezone string is present indicates different semantics about +// / the data (see above). func (rcv *Timestamp) Timezone() []byte { o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) if o != 0 { @@ -177,16 +177,16 @@ func (rcv *Timestamp) Timezone() []byte { return nil } -/// The timezone is an optional string indicating the name of a timezone, -/// one of: -/// -/// * As used in the Olson timezone database (the "tz database" or -/// "tzdata"), such as "America/New_York". -/// * An absolute timezone offset of the form "+XX:XX" or "-XX:XX", -/// such as "+07:30". -/// -/// Whether a timezone string is present indicates different semantics about -/// the data (see above). +// / The timezone is an optional string indicating the name of a timezone, +// / one of: +// / +// / * As used in the Olson timezone database (the "tz database" or +// / "tzdata"), such as "America/New_York". +// / * An absolute timezone offset of the form "+XX:XX" or "-XX:XX", +// / such as "+07:30". +// / +// / Whether a timezone string is present indicates different semantics about +// / the data (see above). func TimestampStart(builder *flatbuffers.Builder) { builder.StartObject(2) } diff --git a/go/arrow/internal/flatbuf/Type.go b/go/arrow/internal/flatbuf/Type.go index ab2bce9c63657..df8ba8650e1cd 100644 --- a/go/arrow/internal/flatbuf/Type.go +++ b/go/arrow/internal/flatbuf/Type.go @@ -20,9 +20,9 @@ package flatbuf import "strconv" -/// ---------------------------------------------------------------------- -/// Top-level Type value, enabling extensible type-specific metadata. We can -/// add new logical types to Type without breaking backwards compatibility +// / ---------------------------------------------------------------------- +// / Top-level Type value, enabling extensible type-specific metadata. We can +// / add new logical types to Type without breaking backwards compatibility type Type byte const ( diff --git a/go/arrow/internal/flatbuf/Union.go b/go/arrow/internal/flatbuf/Union.go index e34121d4757f2..0367fb3c1fb94 100644 --- a/go/arrow/internal/flatbuf/Union.go +++ b/go/arrow/internal/flatbuf/Union.go @@ -22,10 +22,10 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// A union is a complex type with children in Field -/// By default ids in the type vector refer to the offsets in the children -/// optionally typeIds provides an indirection between the child offset and the type id -/// for each child `typeIds[offset]` is the id used in the type vector +// / A union is a complex type with children in Field +// / By default ids in the type vector refer to the offsets in the children +// / optionally typeIds provides an indirection between the child offset and the type id +// / for each child `typeIds[offset]` is the id used in the type vector type Union struct { _tab flatbuffers.Table } diff --git a/go/arrow/internal/flatbuf/Utf8.go b/go/arrow/internal/flatbuf/Utf8.go index 4ff365a37504a..cab4ce7743ca9 100644 --- a/go/arrow/internal/flatbuf/Utf8.go +++ b/go/arrow/internal/flatbuf/Utf8.go @@ -22,7 +22,7 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// Unicode with UTF-8 encoding +// / Unicode with UTF-8 encoding type Utf8 struct { _tab flatbuffers.Table } diff --git a/go/arrow/internal/flatbuf/Utf8View.go b/go/arrow/internal/flatbuf/Utf8View.go index 9cf821490198f..f294126a618b6 100644 --- a/go/arrow/internal/flatbuf/Utf8View.go +++ b/go/arrow/internal/flatbuf/Utf8View.go @@ -22,13 +22,13 @@ import ( flatbuffers "github.com/google/flatbuffers/go" ) -/// Logically the same as Utf8, but the internal representation uses a view -/// struct that contains the string length and either the string's entire data -/// inline (for small strings) or an inlined prefix, an index of another buffer, -/// and an offset pointing to a slice in that buffer (for non-small strings). -/// -/// Since it uses a variable number of data buffers, each Field with this type -/// must have a corresponding entry in `variadicBufferCounts`. +// / Logically the same as Utf8, but the internal representation uses a view +// / struct that contains the string length and either the string's entire data +// / inline (for small strings) or an inlined prefix, an index of another buffer, +// / and an offset pointing to a slice in that buffer (for non-small strings). +// / +// / Since it uses a variable number of data buffers, each Field with this type +// / must have a corresponding entry in `variadicBufferCounts`. type Utf8View struct { _tab flatbuffers.Table } diff --git a/go/arrow/internal/flight_integration/scenario.go b/go/arrow/internal/flight_integration/scenario.go index cde0fff522ec5..ccfc7a0ed45a3 100644 --- a/go/arrow/internal/flight_integration/scenario.go +++ b/go/arrow/internal/flight_integration/scenario.go @@ -2134,7 +2134,7 @@ func (m *flightSqlScenarioTester) ClosePreparedStatement(_ context.Context, requ return nil } -func (m *flightSqlScenarioTester) DoPutPreparedStatementQuery(_ context.Context, cmd flightsql.PreparedStatementQuery, rdr flight.MessageReader, _ flight.MetadataWriter) ([]byte, error){ +func (m *flightSqlScenarioTester) DoPutPreparedStatementQuery(_ context.Context, cmd flightsql.PreparedStatementQuery, rdr flight.MessageReader, _ flight.MetadataWriter) ([]byte, error) { switch string(cmd.GetPreparedStatementHandle()) { case "SELECT PREPARED STATEMENT HANDLE", "SELECT PREPARED STATEMENT WITH TXN HANDLE", diff --git a/go/arrow/ipc/cmd/arrow-cat/main.go b/go/arrow/ipc/cmd/arrow-cat/main.go index 080401e56a83e..4faaabb05ddc1 100644 --- a/go/arrow/ipc/cmd/arrow-cat/main.go +++ b/go/arrow/ipc/cmd/arrow-cat/main.go @@ -18,40 +18,40 @@ // // Examples: // -// $> arrow-cat ./testdata/primitives.data -// version: V4 -// record 1/3... -// col[0] "bools": [true (null) (null) false true] -// col[1] "int8s": [-1 (null) (null) -4 -5] -// col[2] "int16s": [-1 (null) (null) -4 -5] -// col[3] "int32s": [-1 (null) (null) -4 -5] -// col[4] "int64s": [-1 (null) (null) -4 -5] -// col[5] "uint8s": [1 (null) (null) 4 5] -// col[6] "uint16s": [1 (null) (null) 4 5] -// col[7] "uint32s": [1 (null) (null) 4 5] -// col[8] "uint64s": [1 (null) (null) 4 5] -// col[9] "float32s": [1 (null) (null) 4 5] -// col[10] "float64s": [1 (null) (null) 4 5] -// record 2/3... -// col[0] "bools": [true (null) (null) false true] -// [...] +// $> arrow-cat ./testdata/primitives.data +// version: V4 +// record 1/3... +// col[0] "bools": [true (null) (null) false true] +// col[1] "int8s": [-1 (null) (null) -4 -5] +// col[2] "int16s": [-1 (null) (null) -4 -5] +// col[3] "int32s": [-1 (null) (null) -4 -5] +// col[4] "int64s": [-1 (null) (null) -4 -5] +// col[5] "uint8s": [1 (null) (null) 4 5] +// col[6] "uint16s": [1 (null) (null) 4 5] +// col[7] "uint32s": [1 (null) (null) 4 5] +// col[8] "uint64s": [1 (null) (null) 4 5] +// col[9] "float32s": [1 (null) (null) 4 5] +// col[10] "float64s": [1 (null) (null) 4 5] +// record 2/3... +// col[0] "bools": [true (null) (null) false true] +// [...] // -// $> gen-arrow-stream | arrow-cat -// record 1... -// col[0] "bools": [true (null) (null) false true] -// col[1] "int8s": [-1 (null) (null) -4 -5] -// col[2] "int16s": [-1 (null) (null) -4 -5] -// col[3] "int32s": [-1 (null) (null) -4 -5] -// col[4] "int64s": [-1 (null) (null) -4 -5] -// col[5] "uint8s": [1 (null) (null) 4 5] -// col[6] "uint16s": [1 (null) (null) 4 5] -// col[7] "uint32s": [1 (null) (null) 4 5] -// col[8] "uint64s": [1 (null) (null) 4 5] -// col[9] "float32s": [1 (null) (null) 4 5] -// col[10] "float64s": [1 (null) (null) 4 5] -// record 2... -// col[0] "bools": [true (null) (null) false true] -// [...] +// $> gen-arrow-stream | arrow-cat +// record 1... +// col[0] "bools": [true (null) (null) false true] +// col[1] "int8s": [-1 (null) (null) -4 -5] +// col[2] "int16s": [-1 (null) (null) -4 -5] +// col[3] "int32s": [-1 (null) (null) -4 -5] +// col[4] "int64s": [-1 (null) (null) -4 -5] +// col[5] "uint8s": [1 (null) (null) 4 5] +// col[6] "uint16s": [1 (null) (null) 4 5] +// col[7] "uint32s": [1 (null) (null) 4 5] +// col[8] "uint64s": [1 (null) (null) 4 5] +// col[9] "float32s": [1 (null) (null) 4 5] +// col[10] "float64s": [1 (null) (null) 4 5] +// record 2... +// col[0] "bools": [true (null) (null) false true] +// [...] package main import ( diff --git a/go/arrow/ipc/cmd/arrow-ls/main.go b/go/arrow/ipc/cmd/arrow-ls/main.go index 2be1d076e45f0..2f54744c4068d 100644 --- a/go/arrow/ipc/cmd/arrow-ls/main.go +++ b/go/arrow/ipc/cmd/arrow-ls/main.go @@ -18,38 +18,38 @@ // // Examples: // -// $> arrow-ls ./testdata/primitives.data -// version: V4 -// schema: -// fields: 11 -// - bools: type=bool, nullable -// - int8s: type=int8, nullable -// - int16s: type=int16, nullable -// - int32s: type=int32, nullable -// - int64s: type=int64, nullable -// - uint8s: type=uint8, nullable -// - uint16s: type=uint16, nullable -// - uint32s: type=uint32, nullable -// - uint64s: type=uint64, nullable -// - float32s: type=float32, nullable -// - float64s: type=float64, nullable -// records: 3 +// $> arrow-ls ./testdata/primitives.data +// version: V4 +// schema: +// fields: 11 +// - bools: type=bool, nullable +// - int8s: type=int8, nullable +// - int16s: type=int16, nullable +// - int32s: type=int32, nullable +// - int64s: type=int64, nullable +// - uint8s: type=uint8, nullable +// - uint16s: type=uint16, nullable +// - uint32s: type=uint32, nullable +// - uint64s: type=uint64, nullable +// - float32s: type=float32, nullable +// - float64s: type=float64, nullable +// records: 3 // -// $> gen-arrow-stream | arrow-ls -// schema: -// fields: 11 -// - bools: type=bool, nullable -// - int8s: type=int8, nullable -// - int16s: type=int16, nullable -// - int32s: type=int32, nullable -// - int64s: type=int64, nullable -// - uint8s: type=uint8, nullable -// - uint16s: type=uint16, nullable -// - uint32s: type=uint32, nullable -// - uint64s: type=uint64, nullable -// - float32s: type=float32, nullable -// - float64s: type=float64, nullable -// records: 3 +// $> gen-arrow-stream | arrow-ls +// schema: +// fields: 11 +// - bools: type=bool, nullable +// - int8s: type=int8, nullable +// - int16s: type=int16, nullable +// - int32s: type=int32, nullable +// - int64s: type=int64, nullable +// - uint8s: type=uint8, nullable +// - uint16s: type=uint16, nullable +// - uint32s: type=uint32, nullable +// - uint64s: type=uint64, nullable +// - float32s: type=float32, nullable +// - float64s: type=float64, nullable +// records: 3 package main import ( diff --git a/go/arrow/math/math_amd64.go b/go/arrow/math/math_amd64.go index 44301dc2415a5..2397eef718df9 100644 --- a/go/arrow/math/math_amd64.go +++ b/go/arrow/math/math_amd64.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !noasm // +build !noasm package math diff --git a/go/arrow/math/math_arm64.go b/go/arrow/math/math_arm64.go index 014664b046308..b150eb061f9f5 100644 --- a/go/arrow/math/math_arm64.go +++ b/go/arrow/math/math_arm64.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !noasm // +build !noasm package math @@ -25,8 +26,8 @@ import ( func init() { if cpu.ARM64.HasASIMD { initNEON() - } else { - initGo() + } else { + initGo() } } diff --git a/go/arrow/math/math_noasm.go b/go/arrow/math/math_noasm.go index 0fa924d90aa88..5527ebf801891 100644 --- a/go/arrow/math/math_noasm.go +++ b/go/arrow/math/math_noasm.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build noasm // +build noasm package math diff --git a/go/arrow/math/math_ppc64le.go b/go/arrow/math/math_ppc64le.go index 3daeac7efaff8..85c8f2fe2e758 100644 --- a/go/arrow/math/math_ppc64le.go +++ b/go/arrow/math/math_ppc64le.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !noasm // +build !noasm package math diff --git a/go/arrow/math/math_s390x.go b/go/arrow/math/math_s390x.go index 3daeac7efaff8..85c8f2fe2e758 100644 --- a/go/arrow/math/math_s390x.go +++ b/go/arrow/math/math_s390x.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !noasm // +build !noasm package math diff --git a/go/arrow/memory/cgo_allocator.go b/go/arrow/memory/cgo_allocator.go index ffc6b2cb88050..5eb66ade9d861 100644 --- a/go/arrow/memory/cgo_allocator.go +++ b/go/arrow/memory/cgo_allocator.go @@ -14,8 +14,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo -// +build ccalloc +//go:build cgo && ccalloc +// +build cgo,ccalloc package memory diff --git a/go/arrow/memory/cgo_allocator_defaults.go b/go/arrow/memory/cgo_allocator_defaults.go index 501431a0e1eb2..0a2e9a342d37c 100644 --- a/go/arrow/memory/cgo_allocator_defaults.go +++ b/go/arrow/memory/cgo_allocator_defaults.go @@ -14,9 +14,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo -// +build ccalloc -// +build !cclog +//go:build cgo && ccalloc && !cclog +// +build cgo,ccalloc,!cclog package memory diff --git a/go/arrow/memory/cgo_allocator_logging.go b/go/arrow/memory/cgo_allocator_logging.go index 01ad6b394807d..fe2e3a940ce21 100644 --- a/go/arrow/memory/cgo_allocator_logging.go +++ b/go/arrow/memory/cgo_allocator_logging.go @@ -14,9 +14,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo -// +build ccalloc -// +build cclog +//go:build cgo && ccalloc && cclog +// +build cgo,ccalloc,cclog package memory diff --git a/go/arrow/memory/cgo_allocator_test.go b/go/arrow/memory/cgo_allocator_test.go index e7a03767fc89a..4c07cc326c87f 100644 --- a/go/arrow/memory/cgo_allocator_test.go +++ b/go/arrow/memory/cgo_allocator_test.go @@ -14,8 +14,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo -// +build ccalloc +//go:build cgo && ccalloc +// +build cgo,ccalloc package memory diff --git a/go/arrow/memory/memory_amd64.go b/go/arrow/memory/memory_amd64.go index 58356d6482558..895ddc07cf81f 100644 --- a/go/arrow/memory/memory_amd64.go +++ b/go/arrow/memory/memory_amd64.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !noasm // +build !noasm package memory diff --git a/go/arrow/memory/memory_arm64.go b/go/arrow/memory/memory_arm64.go index 3db5d11013164..5260334958526 100755 --- a/go/arrow/memory/memory_arm64.go +++ b/go/arrow/memory/memory_arm64.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !noasm // +build !noasm package memory diff --git a/go/arrow/memory/memory_avx2_amd64.go b/go/arrow/memory/memory_avx2_amd64.go index 2bd851ea53275..39fb3a5f7692f 100644 --- a/go/arrow/memory/memory_avx2_amd64.go +++ b/go/arrow/memory/memory_avx2_amd64.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !noasm // +build !noasm package memory diff --git a/go/arrow/memory/memory_js_wasm.go b/go/arrow/memory/memory_js_wasm.go index 9b94d99ff33ca..5cc0c84d39ee7 100644 --- a/go/arrow/memory/memory_js_wasm.go +++ b/go/arrow/memory/memory_js_wasm.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package memory diff --git a/go/arrow/memory/memory_neon_arm64.go b/go/arrow/memory/memory_neon_arm64.go index 6cb0400c9c597..806ca575f22dd 100755 --- a/go/arrow/memory/memory_neon_arm64.go +++ b/go/arrow/memory/memory_neon_arm64.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !noasm // +build !noasm package memory diff --git a/go/arrow/memory/memory_noasm.go b/go/arrow/memory/memory_noasm.go index bf8846fa2e059..44f19c091c7e0 100644 --- a/go/arrow/memory/memory_noasm.go +++ b/go/arrow/memory/memory_noasm.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build noasm // +build noasm package memory diff --git a/go/arrow/memory/memory_sse4_amd64.go b/go/arrow/memory/memory_sse4_amd64.go index 716c0d2704a88..1711a1ee3eaf7 100644 --- a/go/arrow/memory/memory_sse4_amd64.go +++ b/go/arrow/memory/memory_sse4_amd64.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !noasm // +build !noasm package memory diff --git a/go/internal/utils/min_max_arm64.go b/go/internal/utils/min_max_arm64.go index 7404e95d963e3..d02849665df56 100644 --- a/go/internal/utils/min_max_arm64.go +++ b/go/internal/utils/min_max_arm64.go @@ -21,8 +21,9 @@ package utils import ( "os" "strings" + + "golang.org/x/sys/cpu" ) -import "golang.org/x/sys/cpu" func init() { // Added ability to enable extension via environment: diff --git a/go/parquet/doc.go b/go/parquet/doc.go index ff42be6498d8c..6ab08f83f063f 100644 --- a/go/parquet/doc.go +++ b/go/parquet/doc.go @@ -26,14 +26,15 @@ // This implementation is a native go implementation for reading and writing the // parquet file format. // -// Install +// # Install // // You can download the library and cli utilities via: -// go get -u github.com/apache/arrow/go/v17/parquet -// go install github.com/apache/arrow/go/v17/parquet/cmd/parquet_reader@latest -// go install github.com/apache/arrow/go/v17/parquet/cmd/parquet_schema@latest // -// Modules +// go get -u github.com/apache/arrow/go/v17/parquet +// go install github.com/apache/arrow/go/v17/parquet/cmd/parquet_reader@latest +// go install github.com/apache/arrow/go/v17/parquet/cmd/parquet_schema@latest +// +// # Modules // // This top level parquet package contains the basic common types and reader/writer // properties along with some utilities that are used throughout the other modules. @@ -50,13 +51,13 @@ // The schema module contains the types for manipulating / inspecting / creating // parquet file schemas. // -// Primitive Types +// # Primitive Types // // The Parquet Primitive Types and their corresponding Go types are Boolean (bool), // Int32 (int32), Int64 (int64), Int96 (parquet.Int96), Float (float32), Double (float64), // ByteArray (parquet.ByteArray) and FixedLenByteArray (parquet.FixedLenByteArray). // -// Encodings +// # Encodings // // The encoding types supported in this package are: // Plain, Plain/RLE Dictionary, Delta Binary Packed (only integer types), Delta Byte Array diff --git a/go/parquet/internal/bmi/bitmap_bmi2_amd64.go b/go/parquet/internal/bmi/bitmap_bmi2_amd64.go index ab6dcec40b02b..7fe5a1654911e 100644 --- a/go/parquet/internal/bmi/bitmap_bmi2_amd64.go +++ b/go/parquet/internal/bmi/bitmap_bmi2_amd64.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !noasm // +build !noasm package bmi diff --git a/go/parquet/internal/bmi/bitmap_bmi2_noasm.go b/go/parquet/internal/bmi/bitmap_bmi2_noasm.go index 6dc4a39a60e5a..03be648e011a7 100644 --- a/go/parquet/internal/bmi/bitmap_bmi2_noasm.go +++ b/go/parquet/internal/bmi/bitmap_bmi2_noasm.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build noasm // +build noasm package bmi diff --git a/go/parquet/internal/bmi/bitmap_bmi2_ppc64le.go b/go/parquet/internal/bmi/bitmap_bmi2_ppc64le.go index 498d5452e17ad..60f898f6bd557 100644 --- a/go/parquet/internal/bmi/bitmap_bmi2_ppc64le.go +++ b/go/parquet/internal/bmi/bitmap_bmi2_ppc64le.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !noasm // +build !noasm package bmi diff --git a/go/parquet/internal/bmi/bitmap_bmi2_s390x.go b/go/parquet/internal/bmi/bitmap_bmi2_s390x.go index 498d5452e17ad..60f898f6bd557 100644 --- a/go/parquet/internal/bmi/bitmap_bmi2_s390x.go +++ b/go/parquet/internal/bmi/bitmap_bmi2_s390x.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !noasm // +build !noasm package bmi diff --git a/go/parquet/internal/bmi/bmi_amd64.go b/go/parquet/internal/bmi/bmi_amd64.go index 600ef024f69a8..f894b160d4c8b 100644 --- a/go/parquet/internal/bmi/bmi_amd64.go +++ b/go/parquet/internal/bmi/bmi_amd64.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !noasm // +build !noasm package bmi diff --git a/go/parquet/internal/debug/assert_off.go b/go/parquet/internal/debug/assert_off.go index 52b9a233169d2..1450ecc98a26e 100644 --- a/go/parquet/internal/debug/assert_off.go +++ b/go/parquet/internal/debug/assert_off.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !assert // +build !assert package debug diff --git a/go/parquet/internal/debug/assert_on.go b/go/parquet/internal/debug/assert_on.go index 188e683120466..1a47460fd542a 100644 --- a/go/parquet/internal/debug/assert_on.go +++ b/go/parquet/internal/debug/assert_on.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build assert // +build assert package debug diff --git a/go/parquet/internal/debug/doc.go b/go/parquet/internal/debug/doc.go index 61684d625380d..d3965793a0825 100644 --- a/go/parquet/internal/debug/doc.go +++ b/go/parquet/internal/debug/doc.go @@ -16,7 +16,7 @@ // Package debug provides APIs for conditional runtime assertions and debug logging. // -// Using Assert +// # Using Assert // // To enable runtime assertions, build with the assert tag. When the assert tag is omitted, // the code for the assertion will be omitted from the binary. diff --git a/go/parquet/internal/debug/log_off.go b/go/parquet/internal/debug/log_off.go index 23dcccd810ce4..09f0e09a5ed1d 100644 --- a/go/parquet/internal/debug/log_off.go +++ b/go/parquet/internal/debug/log_off.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !debug // +build !debug package debug diff --git a/go/parquet/internal/debug/log_on.go b/go/parquet/internal/debug/log_on.go index 8d6106099f6f0..0067e442d3693 100644 --- a/go/parquet/internal/debug/log_on.go +++ b/go/parquet/internal/debug/log_on.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build debug // +build debug package debug diff --git a/go/parquet/internal/encoding/delta_byte_array_test.go b/go/parquet/internal/encoding/delta_byte_array_test.go index 1e5e6b2d676ef..c2e4e6849396e 100644 --- a/go/parquet/internal/encoding/delta_byte_array_test.go +++ b/go/parquet/internal/encoding/delta_byte_array_test.go @@ -18,10 +18,11 @@ package encoding import ( "fmt" + "testing" + "github.com/apache/arrow/go/v17/arrow/memory" "github.com/apache/arrow/go/v17/parquet" "github.com/stretchr/testify/assert" - "testing" ) func TestDeltaByteArrayDecoder_SetData(t *testing.T) { diff --git a/go/parquet/internal/gen-go/parquet/GoUnusedProtection__.go b/go/parquet/internal/gen-go/parquet/GoUnusedProtection__.go index 01f1eb5aa99e3..c2a8e5415ed64 100644 --- a/go/parquet/internal/gen-go/parquet/GoUnusedProtection__.go +++ b/go/parquet/internal/gen-go/parquet/GoUnusedProtection__.go @@ -2,5 +2,4 @@ package parquet -var GoUnusedProtection__ int; - +var GoUnusedProtection__ int diff --git a/go/parquet/internal/gen-go/parquet/parquet-consts.go b/go/parquet/internal/gen-go/parquet/parquet-consts.go index ab0a73c596e7d..f83e0be7640ff 100644 --- a/go/parquet/internal/gen-go/parquet/parquet-consts.go +++ b/go/parquet/internal/gen-go/parquet/parquet-consts.go @@ -7,10 +7,11 @@ import ( "context" "errors" "fmt" + "regexp" + "strings" "time" + thrift "github.com/apache/thrift/lib/go/thrift" - "strings" - "regexp" ) // (needed to ensure safety because of naive import list construction.) @@ -20,11 +21,10 @@ var _ = errors.New var _ = context.Background var _ = time.Now var _ = bytes.Equal + // (needed by validator.) var _ = strings.Contains var _ = regexp.MatchString - func init() { } - diff --git a/go/parquet/internal/gen-go/parquet/parquet.go b/go/parquet/internal/gen-go/parquet/parquet.go index 9dcedae8888d3..5b616d1335150 100644 --- a/go/parquet/internal/gen-go/parquet/parquet.go +++ b/go/parquet/internal/gen-go/parquet/parquet.go @@ -8,10 +8,11 @@ import ( "database/sql/driver" "errors" "fmt" + "regexp" + "strings" "time" + thrift "github.com/apache/thrift/lib/go/thrift" - "strings" - "regexp" ) // (needed to ensure safety because of naive import list construction.) @@ -21,1336 +22,1547 @@ var _ = errors.New var _ = context.Background var _ = time.Now var _ = bytes.Equal + // (needed by validator.) var _ = strings.Contains var _ = regexp.MatchString -//Types supported by Parquet. These types are intended to be used in combination -//with the encodings to control the on disk storage format. -//For example INT16 is not included as a type since a good encoding of INT32 -//would handle this. +// Types supported by Parquet. These types are intended to be used in combination +// with the encodings to control the on disk storage format. +// For example INT16 is not included as a type since a good encoding of INT32 +// would handle this. type Type int64 + const ( - Type_BOOLEAN Type = 0 - Type_INT32 Type = 1 - Type_INT64 Type = 2 - Type_INT96 Type = 3 - Type_FLOAT Type = 4 - Type_DOUBLE Type = 5 - Type_BYTE_ARRAY Type = 6 - Type_FIXED_LEN_BYTE_ARRAY Type = 7 + Type_BOOLEAN Type = 0 + Type_INT32 Type = 1 + Type_INT64 Type = 2 + Type_INT96 Type = 3 + Type_FLOAT Type = 4 + Type_DOUBLE Type = 5 + Type_BYTE_ARRAY Type = 6 + Type_FIXED_LEN_BYTE_ARRAY Type = 7 ) func (p Type) String() string { - switch p { - case Type_BOOLEAN: return "BOOLEAN" - case Type_INT32: return "INT32" - case Type_INT64: return "INT64" - case Type_INT96: return "INT96" - case Type_FLOAT: return "FLOAT" - case Type_DOUBLE: return "DOUBLE" - case Type_BYTE_ARRAY: return "BYTE_ARRAY" - case Type_FIXED_LEN_BYTE_ARRAY: return "FIXED_LEN_BYTE_ARRAY" - } - return "" + switch p { + case Type_BOOLEAN: + return "BOOLEAN" + case Type_INT32: + return "INT32" + case Type_INT64: + return "INT64" + case Type_INT96: + return "INT96" + case Type_FLOAT: + return "FLOAT" + case Type_DOUBLE: + return "DOUBLE" + case Type_BYTE_ARRAY: + return "BYTE_ARRAY" + case Type_FIXED_LEN_BYTE_ARRAY: + return "FIXED_LEN_BYTE_ARRAY" + } + return "" } func TypeFromString(s string) (Type, error) { - switch s { - case "BOOLEAN": return Type_BOOLEAN, nil - case "INT32": return Type_INT32, nil - case "INT64": return Type_INT64, nil - case "INT96": return Type_INT96, nil - case "FLOAT": return Type_FLOAT, nil - case "DOUBLE": return Type_DOUBLE, nil - case "BYTE_ARRAY": return Type_BYTE_ARRAY, nil - case "FIXED_LEN_BYTE_ARRAY": return Type_FIXED_LEN_BYTE_ARRAY, nil - } - return Type(0), fmt.Errorf("not a valid Type string") + switch s { + case "BOOLEAN": + return Type_BOOLEAN, nil + case "INT32": + return Type_INT32, nil + case "INT64": + return Type_INT64, nil + case "INT96": + return Type_INT96, nil + case "FLOAT": + return Type_FLOAT, nil + case "DOUBLE": + return Type_DOUBLE, nil + case "BYTE_ARRAY": + return Type_BYTE_ARRAY, nil + case "FIXED_LEN_BYTE_ARRAY": + return Type_FIXED_LEN_BYTE_ARRAY, nil + } + return Type(0), fmt.Errorf("not a valid Type string") } - func TypePtr(v Type) *Type { return &v } func (p Type) MarshalText() ([]byte, error) { -return []byte(p.String()), nil + return []byte(p.String()), nil } func (p *Type) UnmarshalText(text []byte) error { -q, err := TypeFromString(string(text)) -if (err != nil) { -return err -} -*p = q -return nil + q, err := TypeFromString(string(text)) + if err != nil { + return err + } + *p = q + return nil } func (p *Type) Scan(value interface{}) error { -v, ok := value.(int64) -if !ok { -return errors.New("Scan value is not int64") -} -*p = Type(v) -return nil + v, ok := value.(int64) + if !ok { + return errors.New("Scan value is not int64") + } + *p = Type(v) + return nil } -func (p * Type) Value() (driver.Value, error) { - if p == nil { - return nil, nil - } -return int64(*p), nil +func (p *Type) Value() (driver.Value, error) { + if p == nil { + return nil, nil + } + return int64(*p), nil } -//DEPRECATED: Common types used by frameworks(e.g. hive, pig) using parquet. -//ConvertedType is superseded by LogicalType. This enum should not be extended. + +// DEPRECATED: Common types used by frameworks(e.g. hive, pig) using parquet. +// ConvertedType is superseded by LogicalType. This enum should not be extended. // -//See LogicalTypes.md for conversion between ConvertedType and LogicalType. +// See LogicalTypes.md for conversion between ConvertedType and LogicalType. type ConvertedType int64 + const ( - ConvertedType_UTF8 ConvertedType = 0 - ConvertedType_MAP ConvertedType = 1 - ConvertedType_MAP_KEY_VALUE ConvertedType = 2 - ConvertedType_LIST ConvertedType = 3 - ConvertedType_ENUM ConvertedType = 4 - ConvertedType_DECIMAL ConvertedType = 5 - ConvertedType_DATE ConvertedType = 6 - ConvertedType_TIME_MILLIS ConvertedType = 7 - ConvertedType_TIME_MICROS ConvertedType = 8 - ConvertedType_TIMESTAMP_MILLIS ConvertedType = 9 - ConvertedType_TIMESTAMP_MICROS ConvertedType = 10 - ConvertedType_UINT_8 ConvertedType = 11 - ConvertedType_UINT_16 ConvertedType = 12 - ConvertedType_UINT_32 ConvertedType = 13 - ConvertedType_UINT_64 ConvertedType = 14 - ConvertedType_INT_8 ConvertedType = 15 - ConvertedType_INT_16 ConvertedType = 16 - ConvertedType_INT_32 ConvertedType = 17 - ConvertedType_INT_64 ConvertedType = 18 - ConvertedType_JSON ConvertedType = 19 - ConvertedType_BSON ConvertedType = 20 - ConvertedType_INTERVAL ConvertedType = 21 + ConvertedType_UTF8 ConvertedType = 0 + ConvertedType_MAP ConvertedType = 1 + ConvertedType_MAP_KEY_VALUE ConvertedType = 2 + ConvertedType_LIST ConvertedType = 3 + ConvertedType_ENUM ConvertedType = 4 + ConvertedType_DECIMAL ConvertedType = 5 + ConvertedType_DATE ConvertedType = 6 + ConvertedType_TIME_MILLIS ConvertedType = 7 + ConvertedType_TIME_MICROS ConvertedType = 8 + ConvertedType_TIMESTAMP_MILLIS ConvertedType = 9 + ConvertedType_TIMESTAMP_MICROS ConvertedType = 10 + ConvertedType_UINT_8 ConvertedType = 11 + ConvertedType_UINT_16 ConvertedType = 12 + ConvertedType_UINT_32 ConvertedType = 13 + ConvertedType_UINT_64 ConvertedType = 14 + ConvertedType_INT_8 ConvertedType = 15 + ConvertedType_INT_16 ConvertedType = 16 + ConvertedType_INT_32 ConvertedType = 17 + ConvertedType_INT_64 ConvertedType = 18 + ConvertedType_JSON ConvertedType = 19 + ConvertedType_BSON ConvertedType = 20 + ConvertedType_INTERVAL ConvertedType = 21 ) func (p ConvertedType) String() string { - switch p { - case ConvertedType_UTF8: return "UTF8" - case ConvertedType_MAP: return "MAP" - case ConvertedType_MAP_KEY_VALUE: return "MAP_KEY_VALUE" - case ConvertedType_LIST: return "LIST" - case ConvertedType_ENUM: return "ENUM" - case ConvertedType_DECIMAL: return "DECIMAL" - case ConvertedType_DATE: return "DATE" - case ConvertedType_TIME_MILLIS: return "TIME_MILLIS" - case ConvertedType_TIME_MICROS: return "TIME_MICROS" - case ConvertedType_TIMESTAMP_MILLIS: return "TIMESTAMP_MILLIS" - case ConvertedType_TIMESTAMP_MICROS: return "TIMESTAMP_MICROS" - case ConvertedType_UINT_8: return "UINT_8" - case ConvertedType_UINT_16: return "UINT_16" - case ConvertedType_UINT_32: return "UINT_32" - case ConvertedType_UINT_64: return "UINT_64" - case ConvertedType_INT_8: return "INT_8" - case ConvertedType_INT_16: return "INT_16" - case ConvertedType_INT_32: return "INT_32" - case ConvertedType_INT_64: return "INT_64" - case ConvertedType_JSON: return "JSON" - case ConvertedType_BSON: return "BSON" - case ConvertedType_INTERVAL: return "INTERVAL" - } - return "" + switch p { + case ConvertedType_UTF8: + return "UTF8" + case ConvertedType_MAP: + return "MAP" + case ConvertedType_MAP_KEY_VALUE: + return "MAP_KEY_VALUE" + case ConvertedType_LIST: + return "LIST" + case ConvertedType_ENUM: + return "ENUM" + case ConvertedType_DECIMAL: + return "DECIMAL" + case ConvertedType_DATE: + return "DATE" + case ConvertedType_TIME_MILLIS: + return "TIME_MILLIS" + case ConvertedType_TIME_MICROS: + return "TIME_MICROS" + case ConvertedType_TIMESTAMP_MILLIS: + return "TIMESTAMP_MILLIS" + case ConvertedType_TIMESTAMP_MICROS: + return "TIMESTAMP_MICROS" + case ConvertedType_UINT_8: + return "UINT_8" + case ConvertedType_UINT_16: + return "UINT_16" + case ConvertedType_UINT_32: + return "UINT_32" + case ConvertedType_UINT_64: + return "UINT_64" + case ConvertedType_INT_8: + return "INT_8" + case ConvertedType_INT_16: + return "INT_16" + case ConvertedType_INT_32: + return "INT_32" + case ConvertedType_INT_64: + return "INT_64" + case ConvertedType_JSON: + return "JSON" + case ConvertedType_BSON: + return "BSON" + case ConvertedType_INTERVAL: + return "INTERVAL" + } + return "" } func ConvertedTypeFromString(s string) (ConvertedType, error) { - switch s { - case "UTF8": return ConvertedType_UTF8, nil - case "MAP": return ConvertedType_MAP, nil - case "MAP_KEY_VALUE": return ConvertedType_MAP_KEY_VALUE, nil - case "LIST": return ConvertedType_LIST, nil - case "ENUM": return ConvertedType_ENUM, nil - case "DECIMAL": return ConvertedType_DECIMAL, nil - case "DATE": return ConvertedType_DATE, nil - case "TIME_MILLIS": return ConvertedType_TIME_MILLIS, nil - case "TIME_MICROS": return ConvertedType_TIME_MICROS, nil - case "TIMESTAMP_MILLIS": return ConvertedType_TIMESTAMP_MILLIS, nil - case "TIMESTAMP_MICROS": return ConvertedType_TIMESTAMP_MICROS, nil - case "UINT_8": return ConvertedType_UINT_8, nil - case "UINT_16": return ConvertedType_UINT_16, nil - case "UINT_32": return ConvertedType_UINT_32, nil - case "UINT_64": return ConvertedType_UINT_64, nil - case "INT_8": return ConvertedType_INT_8, nil - case "INT_16": return ConvertedType_INT_16, nil - case "INT_32": return ConvertedType_INT_32, nil - case "INT_64": return ConvertedType_INT_64, nil - case "JSON": return ConvertedType_JSON, nil - case "BSON": return ConvertedType_BSON, nil - case "INTERVAL": return ConvertedType_INTERVAL, nil - } - return ConvertedType(0), fmt.Errorf("not a valid ConvertedType string") + switch s { + case "UTF8": + return ConvertedType_UTF8, nil + case "MAP": + return ConvertedType_MAP, nil + case "MAP_KEY_VALUE": + return ConvertedType_MAP_KEY_VALUE, nil + case "LIST": + return ConvertedType_LIST, nil + case "ENUM": + return ConvertedType_ENUM, nil + case "DECIMAL": + return ConvertedType_DECIMAL, nil + case "DATE": + return ConvertedType_DATE, nil + case "TIME_MILLIS": + return ConvertedType_TIME_MILLIS, nil + case "TIME_MICROS": + return ConvertedType_TIME_MICROS, nil + case "TIMESTAMP_MILLIS": + return ConvertedType_TIMESTAMP_MILLIS, nil + case "TIMESTAMP_MICROS": + return ConvertedType_TIMESTAMP_MICROS, nil + case "UINT_8": + return ConvertedType_UINT_8, nil + case "UINT_16": + return ConvertedType_UINT_16, nil + case "UINT_32": + return ConvertedType_UINT_32, nil + case "UINT_64": + return ConvertedType_UINT_64, nil + case "INT_8": + return ConvertedType_INT_8, nil + case "INT_16": + return ConvertedType_INT_16, nil + case "INT_32": + return ConvertedType_INT_32, nil + case "INT_64": + return ConvertedType_INT_64, nil + case "JSON": + return ConvertedType_JSON, nil + case "BSON": + return ConvertedType_BSON, nil + case "INTERVAL": + return ConvertedType_INTERVAL, nil + } + return ConvertedType(0), fmt.Errorf("not a valid ConvertedType string") } - func ConvertedTypePtr(v ConvertedType) *ConvertedType { return &v } func (p ConvertedType) MarshalText() ([]byte, error) { -return []byte(p.String()), nil + return []byte(p.String()), nil } func (p *ConvertedType) UnmarshalText(text []byte) error { -q, err := ConvertedTypeFromString(string(text)) -if (err != nil) { -return err -} -*p = q -return nil + q, err := ConvertedTypeFromString(string(text)) + if err != nil { + return err + } + *p = q + return nil } func (p *ConvertedType) Scan(value interface{}) error { -v, ok := value.(int64) -if !ok { -return errors.New("Scan value is not int64") -} -*p = ConvertedType(v) -return nil + v, ok := value.(int64) + if !ok { + return errors.New("Scan value is not int64") + } + *p = ConvertedType(v) + return nil } -func (p * ConvertedType) Value() (driver.Value, error) { - if p == nil { - return nil, nil - } -return int64(*p), nil +func (p *ConvertedType) Value() (driver.Value, error) { + if p == nil { + return nil, nil + } + return int64(*p), nil } -//Representation of Schemas + +// Representation of Schemas type FieldRepetitionType int64 + const ( - FieldRepetitionType_REQUIRED FieldRepetitionType = 0 - FieldRepetitionType_OPTIONAL FieldRepetitionType = 1 - FieldRepetitionType_REPEATED FieldRepetitionType = 2 + FieldRepetitionType_REQUIRED FieldRepetitionType = 0 + FieldRepetitionType_OPTIONAL FieldRepetitionType = 1 + FieldRepetitionType_REPEATED FieldRepetitionType = 2 ) func (p FieldRepetitionType) String() string { - switch p { - case FieldRepetitionType_REQUIRED: return "REQUIRED" - case FieldRepetitionType_OPTIONAL: return "OPTIONAL" - case FieldRepetitionType_REPEATED: return "REPEATED" - } - return "" + switch p { + case FieldRepetitionType_REQUIRED: + return "REQUIRED" + case FieldRepetitionType_OPTIONAL: + return "OPTIONAL" + case FieldRepetitionType_REPEATED: + return "REPEATED" + } + return "" } func FieldRepetitionTypeFromString(s string) (FieldRepetitionType, error) { - switch s { - case "REQUIRED": return FieldRepetitionType_REQUIRED, nil - case "OPTIONAL": return FieldRepetitionType_OPTIONAL, nil - case "REPEATED": return FieldRepetitionType_REPEATED, nil - } - return FieldRepetitionType(0), fmt.Errorf("not a valid FieldRepetitionType string") + switch s { + case "REQUIRED": + return FieldRepetitionType_REQUIRED, nil + case "OPTIONAL": + return FieldRepetitionType_OPTIONAL, nil + case "REPEATED": + return FieldRepetitionType_REPEATED, nil + } + return FieldRepetitionType(0), fmt.Errorf("not a valid FieldRepetitionType string") } - func FieldRepetitionTypePtr(v FieldRepetitionType) *FieldRepetitionType { return &v } func (p FieldRepetitionType) MarshalText() ([]byte, error) { -return []byte(p.String()), nil + return []byte(p.String()), nil } func (p *FieldRepetitionType) UnmarshalText(text []byte) error { -q, err := FieldRepetitionTypeFromString(string(text)) -if (err != nil) { -return err -} -*p = q -return nil + q, err := FieldRepetitionTypeFromString(string(text)) + if err != nil { + return err + } + *p = q + return nil } func (p *FieldRepetitionType) Scan(value interface{}) error { -v, ok := value.(int64) -if !ok { -return errors.New("Scan value is not int64") -} -*p = FieldRepetitionType(v) -return nil + v, ok := value.(int64) + if !ok { + return errors.New("Scan value is not int64") + } + *p = FieldRepetitionType(v) + return nil } -func (p * FieldRepetitionType) Value() (driver.Value, error) { - if p == nil { - return nil, nil - } -return int64(*p), nil +func (p *FieldRepetitionType) Value() (driver.Value, error) { + if p == nil { + return nil, nil + } + return int64(*p), nil } -//Encodings supported by Parquet. Not all encodings are valid for all types. These -//enums are also used to specify the encoding of definition and repetition levels. -//See the accompanying doc for the details of the more complicated encodings. + +// Encodings supported by Parquet. Not all encodings are valid for all types. These +// enums are also used to specify the encoding of definition and repetition levels. +// See the accompanying doc for the details of the more complicated encodings. type Encoding int64 + const ( - Encoding_PLAIN Encoding = 0 - Encoding_PLAIN_DICTIONARY Encoding = 2 - Encoding_RLE Encoding = 3 - Encoding_BIT_PACKED Encoding = 4 - Encoding_DELTA_BINARY_PACKED Encoding = 5 - Encoding_DELTA_LENGTH_BYTE_ARRAY Encoding = 6 - Encoding_DELTA_BYTE_ARRAY Encoding = 7 - Encoding_RLE_DICTIONARY Encoding = 8 - Encoding_BYTE_STREAM_SPLIT Encoding = 9 + Encoding_PLAIN Encoding = 0 + Encoding_PLAIN_DICTIONARY Encoding = 2 + Encoding_RLE Encoding = 3 + Encoding_BIT_PACKED Encoding = 4 + Encoding_DELTA_BINARY_PACKED Encoding = 5 + Encoding_DELTA_LENGTH_BYTE_ARRAY Encoding = 6 + Encoding_DELTA_BYTE_ARRAY Encoding = 7 + Encoding_RLE_DICTIONARY Encoding = 8 + Encoding_BYTE_STREAM_SPLIT Encoding = 9 ) func (p Encoding) String() string { - switch p { - case Encoding_PLAIN: return "PLAIN" - case Encoding_PLAIN_DICTIONARY: return "PLAIN_DICTIONARY" - case Encoding_RLE: return "RLE" - case Encoding_BIT_PACKED: return "BIT_PACKED" - case Encoding_DELTA_BINARY_PACKED: return "DELTA_BINARY_PACKED" - case Encoding_DELTA_LENGTH_BYTE_ARRAY: return "DELTA_LENGTH_BYTE_ARRAY" - case Encoding_DELTA_BYTE_ARRAY: return "DELTA_BYTE_ARRAY" - case Encoding_RLE_DICTIONARY: return "RLE_DICTIONARY" - case Encoding_BYTE_STREAM_SPLIT: return "BYTE_STREAM_SPLIT" - } - return "" + switch p { + case Encoding_PLAIN: + return "PLAIN" + case Encoding_PLAIN_DICTIONARY: + return "PLAIN_DICTIONARY" + case Encoding_RLE: + return "RLE" + case Encoding_BIT_PACKED: + return "BIT_PACKED" + case Encoding_DELTA_BINARY_PACKED: + return "DELTA_BINARY_PACKED" + case Encoding_DELTA_LENGTH_BYTE_ARRAY: + return "DELTA_LENGTH_BYTE_ARRAY" + case Encoding_DELTA_BYTE_ARRAY: + return "DELTA_BYTE_ARRAY" + case Encoding_RLE_DICTIONARY: + return "RLE_DICTIONARY" + case Encoding_BYTE_STREAM_SPLIT: + return "BYTE_STREAM_SPLIT" + } + return "" } func EncodingFromString(s string) (Encoding, error) { - switch s { - case "PLAIN": return Encoding_PLAIN, nil - case "PLAIN_DICTIONARY": return Encoding_PLAIN_DICTIONARY, nil - case "RLE": return Encoding_RLE, nil - case "BIT_PACKED": return Encoding_BIT_PACKED, nil - case "DELTA_BINARY_PACKED": return Encoding_DELTA_BINARY_PACKED, nil - case "DELTA_LENGTH_BYTE_ARRAY": return Encoding_DELTA_LENGTH_BYTE_ARRAY, nil - case "DELTA_BYTE_ARRAY": return Encoding_DELTA_BYTE_ARRAY, nil - case "RLE_DICTIONARY": return Encoding_RLE_DICTIONARY, nil - case "BYTE_STREAM_SPLIT": return Encoding_BYTE_STREAM_SPLIT, nil - } - return Encoding(0), fmt.Errorf("not a valid Encoding string") + switch s { + case "PLAIN": + return Encoding_PLAIN, nil + case "PLAIN_DICTIONARY": + return Encoding_PLAIN_DICTIONARY, nil + case "RLE": + return Encoding_RLE, nil + case "BIT_PACKED": + return Encoding_BIT_PACKED, nil + case "DELTA_BINARY_PACKED": + return Encoding_DELTA_BINARY_PACKED, nil + case "DELTA_LENGTH_BYTE_ARRAY": + return Encoding_DELTA_LENGTH_BYTE_ARRAY, nil + case "DELTA_BYTE_ARRAY": + return Encoding_DELTA_BYTE_ARRAY, nil + case "RLE_DICTIONARY": + return Encoding_RLE_DICTIONARY, nil + case "BYTE_STREAM_SPLIT": + return Encoding_BYTE_STREAM_SPLIT, nil + } + return Encoding(0), fmt.Errorf("not a valid Encoding string") } - func EncodingPtr(v Encoding) *Encoding { return &v } func (p Encoding) MarshalText() ([]byte, error) { -return []byte(p.String()), nil + return []byte(p.String()), nil } func (p *Encoding) UnmarshalText(text []byte) error { -q, err := EncodingFromString(string(text)) -if (err != nil) { -return err -} -*p = q -return nil + q, err := EncodingFromString(string(text)) + if err != nil { + return err + } + *p = q + return nil } func (p *Encoding) Scan(value interface{}) error { -v, ok := value.(int64) -if !ok { -return errors.New("Scan value is not int64") -} -*p = Encoding(v) -return nil + v, ok := value.(int64) + if !ok { + return errors.New("Scan value is not int64") + } + *p = Encoding(v) + return nil } -func (p * Encoding) Value() (driver.Value, error) { - if p == nil { - return nil, nil - } -return int64(*p), nil +func (p *Encoding) Value() (driver.Value, error) { + if p == nil { + return nil, nil + } + return int64(*p), nil } -//Supported compression algorithms. + +// Supported compression algorithms. // -//Codecs added in format version X.Y can be read by readers based on X.Y and later. -//Codec support may vary between readers based on the format version and -//libraries available at runtime. +// Codecs added in format version X.Y can be read by readers based on X.Y and later. +// Codec support may vary between readers based on the format version and +// libraries available at runtime. // -//See Compression.md for a detailed specification of these algorithms. +// See Compression.md for a detailed specification of these algorithms. type CompressionCodec int64 + const ( - CompressionCodec_UNCOMPRESSED CompressionCodec = 0 - CompressionCodec_SNAPPY CompressionCodec = 1 - CompressionCodec_GZIP CompressionCodec = 2 - CompressionCodec_LZO CompressionCodec = 3 - CompressionCodec_BROTLI CompressionCodec = 4 - CompressionCodec_LZ4 CompressionCodec = 5 - CompressionCodec_ZSTD CompressionCodec = 6 - CompressionCodec_LZ4_RAW CompressionCodec = 7 + CompressionCodec_UNCOMPRESSED CompressionCodec = 0 + CompressionCodec_SNAPPY CompressionCodec = 1 + CompressionCodec_GZIP CompressionCodec = 2 + CompressionCodec_LZO CompressionCodec = 3 + CompressionCodec_BROTLI CompressionCodec = 4 + CompressionCodec_LZ4 CompressionCodec = 5 + CompressionCodec_ZSTD CompressionCodec = 6 + CompressionCodec_LZ4_RAW CompressionCodec = 7 ) func (p CompressionCodec) String() string { - switch p { - case CompressionCodec_UNCOMPRESSED: return "UNCOMPRESSED" - case CompressionCodec_SNAPPY: return "SNAPPY" - case CompressionCodec_GZIP: return "GZIP" - case CompressionCodec_LZO: return "LZO" - case CompressionCodec_BROTLI: return "BROTLI" - case CompressionCodec_LZ4: return "LZ4" - case CompressionCodec_ZSTD: return "ZSTD" - case CompressionCodec_LZ4_RAW: return "LZ4_RAW" - } - return "" + switch p { + case CompressionCodec_UNCOMPRESSED: + return "UNCOMPRESSED" + case CompressionCodec_SNAPPY: + return "SNAPPY" + case CompressionCodec_GZIP: + return "GZIP" + case CompressionCodec_LZO: + return "LZO" + case CompressionCodec_BROTLI: + return "BROTLI" + case CompressionCodec_LZ4: + return "LZ4" + case CompressionCodec_ZSTD: + return "ZSTD" + case CompressionCodec_LZ4_RAW: + return "LZ4_RAW" + } + return "" } func CompressionCodecFromString(s string) (CompressionCodec, error) { - switch s { - case "UNCOMPRESSED": return CompressionCodec_UNCOMPRESSED, nil - case "SNAPPY": return CompressionCodec_SNAPPY, nil - case "GZIP": return CompressionCodec_GZIP, nil - case "LZO": return CompressionCodec_LZO, nil - case "BROTLI": return CompressionCodec_BROTLI, nil - case "LZ4": return CompressionCodec_LZ4, nil - case "ZSTD": return CompressionCodec_ZSTD, nil - case "LZ4_RAW": return CompressionCodec_LZ4_RAW, nil - } - return CompressionCodec(0), fmt.Errorf("not a valid CompressionCodec string") + switch s { + case "UNCOMPRESSED": + return CompressionCodec_UNCOMPRESSED, nil + case "SNAPPY": + return CompressionCodec_SNAPPY, nil + case "GZIP": + return CompressionCodec_GZIP, nil + case "LZO": + return CompressionCodec_LZO, nil + case "BROTLI": + return CompressionCodec_BROTLI, nil + case "LZ4": + return CompressionCodec_LZ4, nil + case "ZSTD": + return CompressionCodec_ZSTD, nil + case "LZ4_RAW": + return CompressionCodec_LZ4_RAW, nil + } + return CompressionCodec(0), fmt.Errorf("not a valid CompressionCodec string") } - func CompressionCodecPtr(v CompressionCodec) *CompressionCodec { return &v } func (p CompressionCodec) MarshalText() ([]byte, error) { -return []byte(p.String()), nil + return []byte(p.String()), nil } func (p *CompressionCodec) UnmarshalText(text []byte) error { -q, err := CompressionCodecFromString(string(text)) -if (err != nil) { -return err -} -*p = q -return nil + q, err := CompressionCodecFromString(string(text)) + if err != nil { + return err + } + *p = q + return nil } func (p *CompressionCodec) Scan(value interface{}) error { -v, ok := value.(int64) -if !ok { -return errors.New("Scan value is not int64") -} -*p = CompressionCodec(v) -return nil + v, ok := value.(int64) + if !ok { + return errors.New("Scan value is not int64") + } + *p = CompressionCodec(v) + return nil } -func (p * CompressionCodec) Value() (driver.Value, error) { - if p == nil { - return nil, nil - } -return int64(*p), nil +func (p *CompressionCodec) Value() (driver.Value, error) { + if p == nil { + return nil, nil + } + return int64(*p), nil } + type PageType int64 + const ( - PageType_DATA_PAGE PageType = 0 - PageType_INDEX_PAGE PageType = 1 - PageType_DICTIONARY_PAGE PageType = 2 - PageType_DATA_PAGE_V2 PageType = 3 + PageType_DATA_PAGE PageType = 0 + PageType_INDEX_PAGE PageType = 1 + PageType_DICTIONARY_PAGE PageType = 2 + PageType_DATA_PAGE_V2 PageType = 3 ) func (p PageType) String() string { - switch p { - case PageType_DATA_PAGE: return "DATA_PAGE" - case PageType_INDEX_PAGE: return "INDEX_PAGE" - case PageType_DICTIONARY_PAGE: return "DICTIONARY_PAGE" - case PageType_DATA_PAGE_V2: return "DATA_PAGE_V2" - } - return "" + switch p { + case PageType_DATA_PAGE: + return "DATA_PAGE" + case PageType_INDEX_PAGE: + return "INDEX_PAGE" + case PageType_DICTIONARY_PAGE: + return "DICTIONARY_PAGE" + case PageType_DATA_PAGE_V2: + return "DATA_PAGE_V2" + } + return "" } func PageTypeFromString(s string) (PageType, error) { - switch s { - case "DATA_PAGE": return PageType_DATA_PAGE, nil - case "INDEX_PAGE": return PageType_INDEX_PAGE, nil - case "DICTIONARY_PAGE": return PageType_DICTIONARY_PAGE, nil - case "DATA_PAGE_V2": return PageType_DATA_PAGE_V2, nil - } - return PageType(0), fmt.Errorf("not a valid PageType string") + switch s { + case "DATA_PAGE": + return PageType_DATA_PAGE, nil + case "INDEX_PAGE": + return PageType_INDEX_PAGE, nil + case "DICTIONARY_PAGE": + return PageType_DICTIONARY_PAGE, nil + case "DATA_PAGE_V2": + return PageType_DATA_PAGE_V2, nil + } + return PageType(0), fmt.Errorf("not a valid PageType string") } - func PageTypePtr(v PageType) *PageType { return &v } func (p PageType) MarshalText() ([]byte, error) { -return []byte(p.String()), nil + return []byte(p.String()), nil } func (p *PageType) UnmarshalText(text []byte) error { -q, err := PageTypeFromString(string(text)) -if (err != nil) { -return err -} -*p = q -return nil + q, err := PageTypeFromString(string(text)) + if err != nil { + return err + } + *p = q + return nil } func (p *PageType) Scan(value interface{}) error { -v, ok := value.(int64) -if !ok { -return errors.New("Scan value is not int64") -} -*p = PageType(v) -return nil + v, ok := value.(int64) + if !ok { + return errors.New("Scan value is not int64") + } + *p = PageType(v) + return nil } -func (p * PageType) Value() (driver.Value, error) { - if p == nil { - return nil, nil - } -return int64(*p), nil +func (p *PageType) Value() (driver.Value, error) { + if p == nil { + return nil, nil + } + return int64(*p), nil } -//Enum to annotate whether lists of min/max elements inside ColumnIndex -//are ordered and if so, in which direction. + +// Enum to annotate whether lists of min/max elements inside ColumnIndex +// are ordered and if so, in which direction. type BoundaryOrder int64 + const ( - BoundaryOrder_UNORDERED BoundaryOrder = 0 - BoundaryOrder_ASCENDING BoundaryOrder = 1 - BoundaryOrder_DESCENDING BoundaryOrder = 2 + BoundaryOrder_UNORDERED BoundaryOrder = 0 + BoundaryOrder_ASCENDING BoundaryOrder = 1 + BoundaryOrder_DESCENDING BoundaryOrder = 2 ) func (p BoundaryOrder) String() string { - switch p { - case BoundaryOrder_UNORDERED: return "UNORDERED" - case BoundaryOrder_ASCENDING: return "ASCENDING" - case BoundaryOrder_DESCENDING: return "DESCENDING" - } - return "" + switch p { + case BoundaryOrder_UNORDERED: + return "UNORDERED" + case BoundaryOrder_ASCENDING: + return "ASCENDING" + case BoundaryOrder_DESCENDING: + return "DESCENDING" + } + return "" } func BoundaryOrderFromString(s string) (BoundaryOrder, error) { - switch s { - case "UNORDERED": return BoundaryOrder_UNORDERED, nil - case "ASCENDING": return BoundaryOrder_ASCENDING, nil - case "DESCENDING": return BoundaryOrder_DESCENDING, nil - } - return BoundaryOrder(0), fmt.Errorf("not a valid BoundaryOrder string") + switch s { + case "UNORDERED": + return BoundaryOrder_UNORDERED, nil + case "ASCENDING": + return BoundaryOrder_ASCENDING, nil + case "DESCENDING": + return BoundaryOrder_DESCENDING, nil + } + return BoundaryOrder(0), fmt.Errorf("not a valid BoundaryOrder string") } - func BoundaryOrderPtr(v BoundaryOrder) *BoundaryOrder { return &v } func (p BoundaryOrder) MarshalText() ([]byte, error) { -return []byte(p.String()), nil + return []byte(p.String()), nil } func (p *BoundaryOrder) UnmarshalText(text []byte) error { -q, err := BoundaryOrderFromString(string(text)) -if (err != nil) { -return err -} -*p = q -return nil + q, err := BoundaryOrderFromString(string(text)) + if err != nil { + return err + } + *p = q + return nil } func (p *BoundaryOrder) Scan(value interface{}) error { -v, ok := value.(int64) -if !ok { -return errors.New("Scan value is not int64") -} -*p = BoundaryOrder(v) -return nil + v, ok := value.(int64) + if !ok { + return errors.New("Scan value is not int64") + } + *p = BoundaryOrder(v) + return nil } -func (p * BoundaryOrder) Value() (driver.Value, error) { - if p == nil { - return nil, nil - } -return int64(*p), nil +func (p *BoundaryOrder) Value() (driver.Value, error) { + if p == nil { + return nil, nil + } + return int64(*p), nil } + // Statistics per row group and per page // All fields are optional. -// +// // Attributes: -// - Max: DEPRECATED: min and max value of the column. Use min_value and max_value. -// +// - Max: DEPRECATED: min and max value of the column. Use min_value and max_value. +// // Values are encoded using PLAIN encoding, except that variable-length byte // arrays do not include a length prefix. -// +// // These fields encode min and max values determined by signed comparison // only. New files should use the correct order for a column's logical type // and store the values in the min_value and max_value fields. -// +// // To support older readers, these may be set when the column order is // signed. -// - Min -// - NullCount: count of null value in the column -// - DistinctCount: count of distinct values occurring -// - MaxValue: Min and max values for the column, determined by its ColumnOrder. -// +// - Min +// - NullCount: count of null value in the column +// - DistinctCount: count of distinct values occurring +// - MaxValue: Min and max values for the column, determined by its ColumnOrder. +// // Values are encoded using PLAIN encoding, except that variable-length byte // arrays do not include a length prefix. -// - MinValue +// - MinValue type Statistics struct { - Max []byte `thrift:"max,1" db:"max" json:"max,omitempty"` - Min []byte `thrift:"min,2" db:"min" json:"min,omitempty"` - NullCount *int64 `thrift:"null_count,3" db:"null_count" json:"null_count,omitempty"` - DistinctCount *int64 `thrift:"distinct_count,4" db:"distinct_count" json:"distinct_count,omitempty"` - MaxValue []byte `thrift:"max_value,5" db:"max_value" json:"max_value,omitempty"` - MinValue []byte `thrift:"min_value,6" db:"min_value" json:"min_value,omitempty"` + Max []byte `thrift:"max,1" db:"max" json:"max,omitempty"` + Min []byte `thrift:"min,2" db:"min" json:"min,omitempty"` + NullCount *int64 `thrift:"null_count,3" db:"null_count" json:"null_count,omitempty"` + DistinctCount *int64 `thrift:"distinct_count,4" db:"distinct_count" json:"distinct_count,omitempty"` + MaxValue []byte `thrift:"max_value,5" db:"max_value" json:"max_value,omitempty"` + MinValue []byte `thrift:"min_value,6" db:"min_value" json:"min_value,omitempty"` } func NewStatistics() *Statistics { - return &Statistics{} + return &Statistics{} } var Statistics_Max_DEFAULT []byte func (p *Statistics) GetMax() []byte { - return p.Max + return p.Max } + var Statistics_Min_DEFAULT []byte func (p *Statistics) GetMin() []byte { - return p.Min + return p.Min } + var Statistics_NullCount_DEFAULT int64 + func (p *Statistics) GetNullCount() int64 { - if !p.IsSetNullCount() { - return Statistics_NullCount_DEFAULT - } -return *p.NullCount + if !p.IsSetNullCount() { + return Statistics_NullCount_DEFAULT + } + return *p.NullCount } + var Statistics_DistinctCount_DEFAULT int64 + func (p *Statistics) GetDistinctCount() int64 { - if !p.IsSetDistinctCount() { - return Statistics_DistinctCount_DEFAULT - } -return *p.DistinctCount + if !p.IsSetDistinctCount() { + return Statistics_DistinctCount_DEFAULT + } + return *p.DistinctCount } + var Statistics_MaxValue_DEFAULT []byte func (p *Statistics) GetMaxValue() []byte { - return p.MaxValue + return p.MaxValue } + var Statistics_MinValue_DEFAULT []byte func (p *Statistics) GetMinValue() []byte { - return p.MinValue + return p.MinValue } func (p *Statistics) IsSetMax() bool { - return p.Max != nil + return p.Max != nil } func (p *Statistics) IsSetMin() bool { - return p.Min != nil + return p.Min != nil } func (p *Statistics) IsSetNullCount() bool { - return p.NullCount != nil + return p.NullCount != nil } func (p *Statistics) IsSetDistinctCount() bool { - return p.DistinctCount != nil + return p.DistinctCount != nil } func (p *Statistics) IsSetMaxValue() bool { - return p.MaxValue != nil + return p.MaxValue != nil } func (p *Statistics) IsSetMinValue() bool { - return p.MinValue != nil + return p.MinValue != nil } func (p *Statistics) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.STRING { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.STRING { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 3: - if fieldTypeId == thrift.I64 { - if err := p.ReadField3(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 4: - if fieldTypeId == thrift.I64 { - if err := p.ReadField4(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 5: - if fieldTypeId == thrift.STRING { - if err := p.ReadField5(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 6: - if fieldTypeId == thrift.STRING { - if err := p.ReadField6(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil -} - -func (p *Statistics) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadBinary(ctx); err != nil { - return thrift.PrependError("error reading field 1: ", err) -} else { - p.Max = v -} - return nil -} - -func (p *Statistics) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadBinary(ctx); err != nil { - return thrift.PrependError("error reading field 2: ", err) -} else { - p.Min = v -} - return nil -} - -func (p *Statistics) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(ctx); err != nil { - return thrift.PrependError("error reading field 3: ", err) -} else { - p.NullCount = &v -} - return nil -} - -func (p *Statistics) ReadField4(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(ctx); err != nil { - return thrift.PrependError("error reading field 4: ", err) -} else { - p.DistinctCount = &v -} - return nil -} - -func (p *Statistics) ReadField5(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadBinary(ctx); err != nil { - return thrift.PrependError("error reading field 5: ", err) -} else { - p.MaxValue = v -} - return nil -} - -func (p *Statistics) ReadField6(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadBinary(ctx); err != nil { - return thrift.PrependError("error reading field 6: ", err) -} else { - p.MinValue = v -} - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.STRING { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.STRING { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 3: + if fieldTypeId == thrift.I64 { + if err := p.ReadField3(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 4: + if fieldTypeId == thrift.I64 { + if err := p.ReadField4(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 5: + if fieldTypeId == thrift.STRING { + if err := p.ReadField5(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 6: + if fieldTypeId == thrift.STRING { + if err := p.ReadField6(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil +} + +func (p *Statistics) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadBinary(ctx); err != nil { + return thrift.PrependError("error reading field 1: ", err) + } else { + p.Max = v + } + return nil +} + +func (p *Statistics) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadBinary(ctx); err != nil { + return thrift.PrependError("error reading field 2: ", err) + } else { + p.Min = v + } + return nil +} + +func (p *Statistics) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI64(ctx); err != nil { + return thrift.PrependError("error reading field 3: ", err) + } else { + p.NullCount = &v + } + return nil +} + +func (p *Statistics) ReadField4(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI64(ctx); err != nil { + return thrift.PrependError("error reading field 4: ", err) + } else { + p.DistinctCount = &v + } + return nil +} + +func (p *Statistics) ReadField5(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadBinary(ctx); err != nil { + return thrift.PrependError("error reading field 5: ", err) + } else { + p.MaxValue = v + } + return nil +} + +func (p *Statistics) ReadField6(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadBinary(ctx); err != nil { + return thrift.PrependError("error reading field 6: ", err) + } else { + p.MinValue = v + } + return nil } func (p *Statistics) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "Statistics"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - if err := p.writeField3(ctx, oprot); err != nil { return err } - if err := p.writeField4(ctx, oprot); err != nil { return err } - if err := p.writeField5(ctx, oprot); err != nil { return err } - if err := p.writeField6(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "Statistics"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + if err := p.writeField3(ctx, oprot); err != nil { + return err + } + if err := p.writeField4(ctx, oprot); err != nil { + return err + } + if err := p.writeField5(ctx, oprot); err != nil { + return err + } + if err := p.writeField6(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *Statistics) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetMax() { - if err := oprot.WriteFieldBegin(ctx, "max", thrift.STRING, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:max: ", p), err) } - if err := oprot.WriteBinary(ctx, p.Max); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.max (1) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:max: ", p), err) } - } - return err + if p.IsSetMax() { + if err := oprot.WriteFieldBegin(ctx, "max", thrift.STRING, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:max: ", p), err) + } + if err := oprot.WriteBinary(ctx, p.Max); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.max (1) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:max: ", p), err) + } + } + return err } func (p *Statistics) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetMin() { - if err := oprot.WriteFieldBegin(ctx, "min", thrift.STRING, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:min: ", p), err) } - if err := oprot.WriteBinary(ctx, p.Min); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.min (2) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:min: ", p), err) } - } - return err + if p.IsSetMin() { + if err := oprot.WriteFieldBegin(ctx, "min", thrift.STRING, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:min: ", p), err) + } + if err := oprot.WriteBinary(ctx, p.Min); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.min (2) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:min: ", p), err) + } + } + return err } func (p *Statistics) writeField3(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetNullCount() { - if err := oprot.WriteFieldBegin(ctx, "null_count", thrift.I64, 3); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:null_count: ", p), err) } - if err := oprot.WriteI64(ctx, int64(*p.NullCount)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.null_count (3) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 3:null_count: ", p), err) } - } - return err + if p.IsSetNullCount() { + if err := oprot.WriteFieldBegin(ctx, "null_count", thrift.I64, 3); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:null_count: ", p), err) + } + if err := oprot.WriteI64(ctx, int64(*p.NullCount)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.null_count (3) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 3:null_count: ", p), err) + } + } + return err } func (p *Statistics) writeField4(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetDistinctCount() { - if err := oprot.WriteFieldBegin(ctx, "distinct_count", thrift.I64, 4); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:distinct_count: ", p), err) } - if err := oprot.WriteI64(ctx, int64(*p.DistinctCount)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.distinct_count (4) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 4:distinct_count: ", p), err) } - } - return err + if p.IsSetDistinctCount() { + if err := oprot.WriteFieldBegin(ctx, "distinct_count", thrift.I64, 4); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:distinct_count: ", p), err) + } + if err := oprot.WriteI64(ctx, int64(*p.DistinctCount)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.distinct_count (4) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 4:distinct_count: ", p), err) + } + } + return err } func (p *Statistics) writeField5(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetMaxValue() { - if err := oprot.WriteFieldBegin(ctx, "max_value", thrift.STRING, 5); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:max_value: ", p), err) } - if err := oprot.WriteBinary(ctx, p.MaxValue); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.max_value (5) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 5:max_value: ", p), err) } - } - return err + if p.IsSetMaxValue() { + if err := oprot.WriteFieldBegin(ctx, "max_value", thrift.STRING, 5); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:max_value: ", p), err) + } + if err := oprot.WriteBinary(ctx, p.MaxValue); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.max_value (5) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 5:max_value: ", p), err) + } + } + return err } func (p *Statistics) writeField6(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetMinValue() { - if err := oprot.WriteFieldBegin(ctx, "min_value", thrift.STRING, 6); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 6:min_value: ", p), err) } - if err := oprot.WriteBinary(ctx, p.MinValue); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.min_value (6) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 6:min_value: ", p), err) } - } - return err + if p.IsSetMinValue() { + if err := oprot.WriteFieldBegin(ctx, "min_value", thrift.STRING, 6); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 6:min_value: ", p), err) + } + if err := oprot.WriteBinary(ctx, p.MinValue); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.min_value (6) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 6:min_value: ", p), err) + } + } + return err } func (p *Statistics) Equals(other *Statistics) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if bytes.Compare(p.Max, other.Max) != 0 { return false } - if bytes.Compare(p.Min, other.Min) != 0 { return false } - if p.NullCount != other.NullCount { - if p.NullCount == nil || other.NullCount == nil { - return false - } - if (*p.NullCount) != (*other.NullCount) { return false } - } - if p.DistinctCount != other.DistinctCount { - if p.DistinctCount == nil || other.DistinctCount == nil { - return false - } - if (*p.DistinctCount) != (*other.DistinctCount) { return false } - } - if bytes.Compare(p.MaxValue, other.MaxValue) != 0 { return false } - if bytes.Compare(p.MinValue, other.MinValue) != 0 { return false } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if bytes.Compare(p.Max, other.Max) != 0 { + return false + } + if bytes.Compare(p.Min, other.Min) != 0 { + return false + } + if p.NullCount != other.NullCount { + if p.NullCount == nil || other.NullCount == nil { + return false + } + if (*p.NullCount) != (*other.NullCount) { + return false + } + } + if p.DistinctCount != other.DistinctCount { + if p.DistinctCount == nil || other.DistinctCount == nil { + return false + } + if (*p.DistinctCount) != (*other.DistinctCount) { + return false + } + } + if bytes.Compare(p.MaxValue, other.MaxValue) != 0 { + return false + } + if bytes.Compare(p.MinValue, other.MinValue) != 0 { + return false + } + return true } func (p *Statistics) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("Statistics(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("Statistics(%+v)", *p) } func (p *Statistics) Validate() error { - return nil + return nil } + // Empty structs to use as logical type annotations type StringType struct { } func NewStringType() *StringType { - return &StringType{} + return &StringType{} } func (p *StringType) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil } func (p *StringType) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "StringType"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "StringType"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *StringType) Equals(other *StringType) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + return true } func (p *StringType) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("StringType(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("StringType(%+v)", *p) } func (p *StringType) Validate() error { - return nil + return nil } + type UUIDType struct { } func NewUUIDType() *UUIDType { - return &UUIDType{} + return &UUIDType{} } func (p *UUIDType) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil } func (p *UUIDType) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "UUIDType"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "UUIDType"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *UUIDType) Equals(other *UUIDType) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + return true } func (p *UUIDType) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("UUIDType(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("UUIDType(%+v)", *p) } func (p *UUIDType) Validate() error { - return nil + return nil } + type MapType struct { } func NewMapType() *MapType { - return &MapType{} + return &MapType{} } func (p *MapType) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil } func (p *MapType) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "MapType"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "MapType"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *MapType) Equals(other *MapType) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + return true } func (p *MapType) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("MapType(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("MapType(%+v)", *p) } func (p *MapType) Validate() error { - return nil + return nil } + type ListType struct { } func NewListType() *ListType { - return &ListType{} + return &ListType{} } func (p *ListType) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil } func (p *ListType) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "ListType"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "ListType"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *ListType) Equals(other *ListType) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + return true } func (p *ListType) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("ListType(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("ListType(%+v)", *p) } func (p *ListType) Validate() error { - return nil + return nil } + type EnumType struct { } func NewEnumType() *EnumType { - return &EnumType{} + return &EnumType{} } func (p *EnumType) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil } func (p *EnumType) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "EnumType"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "EnumType"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *EnumType) Equals(other *EnumType) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + return true } func (p *EnumType) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("EnumType(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("EnumType(%+v)", *p) } func (p *EnumType) Validate() error { - return nil + return nil } + type DateType struct { } func NewDateType() *DateType { - return &DateType{} + return &DateType{} } func (p *DateType) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil } func (p *DateType) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "DateType"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "DateType"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *DateType) Equals(other *DateType) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + return true } func (p *DateType) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("DateType(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("DateType(%+v)", *p) } func (p *DateType) Validate() error { - return nil + return nil } + type Float16Type struct { } func NewFloat16Type() *Float16Type { - return &Float16Type{} + return &Float16Type{} } func (p *Float16Type) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil } func (p *Float16Type) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "Float16Type"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "Float16Type"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *Float16Type) Equals(other *Float16Type) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + return true } func (p *Float16Type) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("Float16Type(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("Float16Type(%+v)", *p) } func (p *Float16Type) Validate() error { - return nil + return nil } + // Logical type to annotate a column that is always null. -// +// // Sometimes when discovering the schema of existing data, values are always // null and the physical type can't be determined. This annotation signals // the case where the physical type was guessed from all null values. @@ -1358,8169 +1570,9269 @@ type NullType struct { } func NewNullType() *NullType { - return &NullType{} + return &NullType{} } func (p *NullType) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil } func (p *NullType) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "NullType"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "NullType"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *NullType) Equals(other *NullType) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + return true } func (p *NullType) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("NullType(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("NullType(%+v)", *p) } func (p *NullType) Validate() error { - return nil + return nil } + // Decimal logical type annotation -// +// // To maintain forward-compatibility in v1, implementations using this logical // type must also set scale and precision on the annotated SchemaElement. -// +// // Allowed for physical types: INT32, INT64, FIXED, and BINARY -// +// // Attributes: -// - Scale -// - Precision +// - Scale +// - Precision type DecimalType struct { - Scale int32 `thrift:"scale,1,required" db:"scale" json:"scale"` - Precision int32 `thrift:"precision,2,required" db:"precision" json:"precision"` + Scale int32 `thrift:"scale,1,required" db:"scale" json:"scale"` + Precision int32 `thrift:"precision,2,required" db:"precision" json:"precision"` } func NewDecimalType() *DecimalType { - return &DecimalType{} + return &DecimalType{} } - func (p *DecimalType) GetScale() int32 { - return p.Scale + return p.Scale } func (p *DecimalType) GetPrecision() int32 { - return p.Precision + return p.Precision } func (p *DecimalType) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - var issetScale bool = false; - var issetPrecision bool = false; - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.I32 { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - issetScale = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.I32 { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - issetPrecision = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - if !issetScale{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Scale is not set")); - } - if !issetPrecision{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Precision is not set")); - } - return nil -} - -func (p *DecimalType) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 1: ", err) -} else { - p.Scale = v -} - return nil -} - -func (p *DecimalType) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 2: ", err) -} else { - p.Precision = v -} - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + var issetScale bool = false + var issetPrecision bool = false + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.I32 { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + issetScale = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.I32 { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + issetPrecision = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + if !issetScale { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Scale is not set")) + } + if !issetPrecision { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Precision is not set")) + } + return nil +} + +func (p *DecimalType) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 1: ", err) + } else { + p.Scale = v + } + return nil +} + +func (p *DecimalType) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 2: ", err) + } else { + p.Precision = v + } + return nil } func (p *DecimalType) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "DecimalType"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "DecimalType"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *DecimalType) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "scale", thrift.I32, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:scale: ", p), err) } - if err := oprot.WriteI32(ctx, int32(p.Scale)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.scale (1) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:scale: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "scale", thrift.I32, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:scale: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.Scale)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.scale (1) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:scale: ", p), err) + } + return err } func (p *DecimalType) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "precision", thrift.I32, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:precision: ", p), err) } - if err := oprot.WriteI32(ctx, int32(p.Precision)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.precision (2) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:precision: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "precision", thrift.I32, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:precision: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.Precision)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.precision (2) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:precision: ", p), err) + } + return err } func (p *DecimalType) Equals(other *DecimalType) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if p.Scale != other.Scale { return false } - if p.Precision != other.Precision { return false } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if p.Scale != other.Scale { + return false + } + if p.Precision != other.Precision { + return false + } + return true } func (p *DecimalType) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("DecimalType(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("DecimalType(%+v)", *p) } func (p *DecimalType) Validate() error { - return nil + return nil } + // Time units for logical types type MilliSeconds struct { } func NewMilliSeconds() *MilliSeconds { - return &MilliSeconds{} + return &MilliSeconds{} } func (p *MilliSeconds) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil } func (p *MilliSeconds) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "MilliSeconds"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "MilliSeconds"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *MilliSeconds) Equals(other *MilliSeconds) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + return true } func (p *MilliSeconds) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("MilliSeconds(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("MilliSeconds(%+v)", *p) } func (p *MilliSeconds) Validate() error { - return nil + return nil } + type MicroSeconds struct { } func NewMicroSeconds() *MicroSeconds { - return &MicroSeconds{} + return &MicroSeconds{} } func (p *MicroSeconds) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil } func (p *MicroSeconds) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "MicroSeconds"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "MicroSeconds"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *MicroSeconds) Equals(other *MicroSeconds) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + return true } func (p *MicroSeconds) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("MicroSeconds(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("MicroSeconds(%+v)", *p) } func (p *MicroSeconds) Validate() error { - return nil + return nil } + type NanoSeconds struct { } func NewNanoSeconds() *NanoSeconds { - return &NanoSeconds{} + return &NanoSeconds{} } func (p *NanoSeconds) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil } func (p *NanoSeconds) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "NanoSeconds"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "NanoSeconds"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *NanoSeconds) Equals(other *NanoSeconds) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + return true } func (p *NanoSeconds) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("NanoSeconds(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("NanoSeconds(%+v)", *p) } func (p *NanoSeconds) Validate() error { - return nil + return nil } + // Attributes: -// - MILLIS -// - MICROS -// - NANOS +// - MILLIS +// - MICROS +// - NANOS type TimeUnit struct { - MILLIS *MilliSeconds `thrift:"MILLIS,1" db:"MILLIS" json:"MILLIS,omitempty"` - MICROS *MicroSeconds `thrift:"MICROS,2" db:"MICROS" json:"MICROS,omitempty"` - NANOS *NanoSeconds `thrift:"NANOS,3" db:"NANOS" json:"NANOS,omitempty"` + MILLIS *MilliSeconds `thrift:"MILLIS,1" db:"MILLIS" json:"MILLIS,omitempty"` + MICROS *MicroSeconds `thrift:"MICROS,2" db:"MICROS" json:"MICROS,omitempty"` + NANOS *NanoSeconds `thrift:"NANOS,3" db:"NANOS" json:"NANOS,omitempty"` } func NewTimeUnit() *TimeUnit { - return &TimeUnit{} + return &TimeUnit{} } var TimeUnit_MILLIS_DEFAULT *MilliSeconds + func (p *TimeUnit) GetMILLIS() *MilliSeconds { - if !p.IsSetMILLIS() { - return TimeUnit_MILLIS_DEFAULT - } -return p.MILLIS + if !p.IsSetMILLIS() { + return TimeUnit_MILLIS_DEFAULT + } + return p.MILLIS } + var TimeUnit_MICROS_DEFAULT *MicroSeconds + func (p *TimeUnit) GetMICROS() *MicroSeconds { - if !p.IsSetMICROS() { - return TimeUnit_MICROS_DEFAULT - } -return p.MICROS + if !p.IsSetMICROS() { + return TimeUnit_MICROS_DEFAULT + } + return p.MICROS } + var TimeUnit_NANOS_DEFAULT *NanoSeconds + func (p *TimeUnit) GetNANOS() *NanoSeconds { - if !p.IsSetNANOS() { - return TimeUnit_NANOS_DEFAULT - } -return p.NANOS + if !p.IsSetNANOS() { + return TimeUnit_NANOS_DEFAULT + } + return p.NANOS } func (p *TimeUnit) CountSetFieldsTimeUnit() int { - count := 0 - if (p.IsSetMILLIS()) { - count++ - } - if (p.IsSetMICROS()) { - count++ - } - if (p.IsSetNANOS()) { - count++ - } - return count + count := 0 + if p.IsSetMILLIS() { + count++ + } + if p.IsSetMICROS() { + count++ + } + if p.IsSetNANOS() { + count++ + } + return count } func (p *TimeUnit) IsSetMILLIS() bool { - return p.MILLIS != nil + return p.MILLIS != nil } func (p *TimeUnit) IsSetMICROS() bool { - return p.MICROS != nil + return p.MICROS != nil } func (p *TimeUnit) IsSetNANOS() bool { - return p.NANOS != nil + return p.NANOS != nil } func (p *TimeUnit) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 3: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField3(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil -} - -func (p *TimeUnit) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - p.MILLIS = &MilliSeconds{} - if err := p.MILLIS.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.MILLIS), err) - } - return nil -} - -func (p *TimeUnit) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - p.MICROS = &MicroSeconds{} - if err := p.MICROS.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.MICROS), err) - } - return nil -} - -func (p *TimeUnit) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { - p.NANOS = &NanoSeconds{} - if err := p.NANOS.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.NANOS), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 3: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField3(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil +} + +func (p *TimeUnit) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + p.MILLIS = &MilliSeconds{} + if err := p.MILLIS.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.MILLIS), err) + } + return nil +} + +func (p *TimeUnit) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + p.MICROS = &MicroSeconds{} + if err := p.MICROS.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.MICROS), err) + } + return nil +} + +func (p *TimeUnit) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { + p.NANOS = &NanoSeconds{} + if err := p.NANOS.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.NANOS), err) + } + return nil } func (p *TimeUnit) Write(ctx context.Context, oprot thrift.TProtocol) error { - if c := p.CountSetFieldsTimeUnit(); c != 1 { - return fmt.Errorf("%T write union: exactly one field must be set (%d set)", p, c) - } - if err := oprot.WriteStructBegin(ctx, "TimeUnit"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - if err := p.writeField3(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if c := p.CountSetFieldsTimeUnit(); c != 1 { + return fmt.Errorf("%T write union: exactly one field must be set (%d set)", p, c) + } + if err := oprot.WriteStructBegin(ctx, "TimeUnit"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + if err := p.writeField3(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *TimeUnit) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetMILLIS() { - if err := oprot.WriteFieldBegin(ctx, "MILLIS", thrift.STRUCT, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:MILLIS: ", p), err) } - if err := p.MILLIS.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.MILLIS), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:MILLIS: ", p), err) } - } - return err + if p.IsSetMILLIS() { + if err := oprot.WriteFieldBegin(ctx, "MILLIS", thrift.STRUCT, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:MILLIS: ", p), err) + } + if err := p.MILLIS.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.MILLIS), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:MILLIS: ", p), err) + } + } + return err } func (p *TimeUnit) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetMICROS() { - if err := oprot.WriteFieldBegin(ctx, "MICROS", thrift.STRUCT, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:MICROS: ", p), err) } - if err := p.MICROS.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.MICROS), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:MICROS: ", p), err) } - } - return err + if p.IsSetMICROS() { + if err := oprot.WriteFieldBegin(ctx, "MICROS", thrift.STRUCT, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:MICROS: ", p), err) + } + if err := p.MICROS.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.MICROS), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:MICROS: ", p), err) + } + } + return err } func (p *TimeUnit) writeField3(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetNANOS() { - if err := oprot.WriteFieldBegin(ctx, "NANOS", thrift.STRUCT, 3); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:NANOS: ", p), err) } - if err := p.NANOS.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.NANOS), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 3:NANOS: ", p), err) } - } - return err + if p.IsSetNANOS() { + if err := oprot.WriteFieldBegin(ctx, "NANOS", thrift.STRUCT, 3); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:NANOS: ", p), err) + } + if err := p.NANOS.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.NANOS), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 3:NANOS: ", p), err) + } + } + return err } func (p *TimeUnit) Equals(other *TimeUnit) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if !p.MILLIS.Equals(other.MILLIS) { return false } - if !p.MICROS.Equals(other.MICROS) { return false } - if !p.NANOS.Equals(other.NANOS) { return false } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if !p.MILLIS.Equals(other.MILLIS) { + return false + } + if !p.MICROS.Equals(other.MICROS) { + return false + } + if !p.NANOS.Equals(other.NANOS) { + return false + } + return true } func (p *TimeUnit) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("TimeUnit(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("TimeUnit(%+v)", *p) } func (p *TimeUnit) Validate() error { - return nil + return nil } + // Timestamp logical type annotation -// +// // Allowed for physical types: INT64 -// +// // Attributes: -// - IsAdjustedToUTC -// - Unit +// - IsAdjustedToUTC +// - Unit type TimestampType struct { - IsAdjustedToUTC bool `thrift:"isAdjustedToUTC,1,required" db:"isAdjustedToUTC" json:"isAdjustedToUTC"` - Unit *TimeUnit `thrift:"unit,2,required" db:"unit" json:"unit"` + IsAdjustedToUTC bool `thrift:"isAdjustedToUTC,1,required" db:"isAdjustedToUTC" json:"isAdjustedToUTC"` + Unit *TimeUnit `thrift:"unit,2,required" db:"unit" json:"unit"` } func NewTimestampType() *TimestampType { - return &TimestampType{} + return &TimestampType{} } - func (p *TimestampType) GetIsAdjustedToUTC() bool { - return p.IsAdjustedToUTC + return p.IsAdjustedToUTC } + var TimestampType_Unit_DEFAULT *TimeUnit + func (p *TimestampType) GetUnit() *TimeUnit { - if !p.IsSetUnit() { - return TimestampType_Unit_DEFAULT - } -return p.Unit + if !p.IsSetUnit() { + return TimestampType_Unit_DEFAULT + } + return p.Unit } func (p *TimestampType) IsSetUnit() bool { - return p.Unit != nil + return p.Unit != nil } func (p *TimestampType) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - var issetIsAdjustedToUTC bool = false; - var issetUnit bool = false; - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.BOOL { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - issetIsAdjustedToUTC = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - issetUnit = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - if !issetIsAdjustedToUTC{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field IsAdjustedToUTC is not set")); - } - if !issetUnit{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Unit is not set")); - } - return nil -} - -func (p *TimestampType) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadBool(ctx); err != nil { - return thrift.PrependError("error reading field 1: ", err) -} else { - p.IsAdjustedToUTC = v -} - return nil -} - -func (p *TimestampType) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - p.Unit = &TimeUnit{} - if err := p.Unit.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.Unit), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + var issetIsAdjustedToUTC bool = false + var issetUnit bool = false + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.BOOL { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + issetIsAdjustedToUTC = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + issetUnit = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + if !issetIsAdjustedToUTC { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field IsAdjustedToUTC is not set")) + } + if !issetUnit { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Unit is not set")) + } + return nil +} + +func (p *TimestampType) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadBool(ctx); err != nil { + return thrift.PrependError("error reading field 1: ", err) + } else { + p.IsAdjustedToUTC = v + } + return nil +} + +func (p *TimestampType) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + p.Unit = &TimeUnit{} + if err := p.Unit.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.Unit), err) + } + return nil } func (p *TimestampType) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "TimestampType"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "TimestampType"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *TimestampType) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "isAdjustedToUTC", thrift.BOOL, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:isAdjustedToUTC: ", p), err) } - if err := oprot.WriteBool(ctx, bool(p.IsAdjustedToUTC)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.isAdjustedToUTC (1) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:isAdjustedToUTC: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "isAdjustedToUTC", thrift.BOOL, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:isAdjustedToUTC: ", p), err) + } + if err := oprot.WriteBool(ctx, bool(p.IsAdjustedToUTC)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.isAdjustedToUTC (1) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:isAdjustedToUTC: ", p), err) + } + return err } func (p *TimestampType) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "unit", thrift.STRUCT, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:unit: ", p), err) } - if err := p.Unit.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.Unit), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:unit: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "unit", thrift.STRUCT, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:unit: ", p), err) + } + if err := p.Unit.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.Unit), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:unit: ", p), err) + } + return err } func (p *TimestampType) Equals(other *TimestampType) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if p.IsAdjustedToUTC != other.IsAdjustedToUTC { return false } - if !p.Unit.Equals(other.Unit) { return false } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if p.IsAdjustedToUTC != other.IsAdjustedToUTC { + return false + } + if !p.Unit.Equals(other.Unit) { + return false + } + return true } func (p *TimestampType) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("TimestampType(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("TimestampType(%+v)", *p) } func (p *TimestampType) Validate() error { - return nil + return nil } + // Time logical type annotation -// +// // Allowed for physical types: INT32 (millis), INT64 (micros, nanos) -// +// // Attributes: -// - IsAdjustedToUTC -// - Unit +// - IsAdjustedToUTC +// - Unit type TimeType struct { - IsAdjustedToUTC bool `thrift:"isAdjustedToUTC,1,required" db:"isAdjustedToUTC" json:"isAdjustedToUTC"` - Unit *TimeUnit `thrift:"unit,2,required" db:"unit" json:"unit"` + IsAdjustedToUTC bool `thrift:"isAdjustedToUTC,1,required" db:"isAdjustedToUTC" json:"isAdjustedToUTC"` + Unit *TimeUnit `thrift:"unit,2,required" db:"unit" json:"unit"` } func NewTimeType() *TimeType { - return &TimeType{} + return &TimeType{} } - func (p *TimeType) GetIsAdjustedToUTC() bool { - return p.IsAdjustedToUTC + return p.IsAdjustedToUTC } + var TimeType_Unit_DEFAULT *TimeUnit + func (p *TimeType) GetUnit() *TimeUnit { - if !p.IsSetUnit() { - return TimeType_Unit_DEFAULT - } -return p.Unit + if !p.IsSetUnit() { + return TimeType_Unit_DEFAULT + } + return p.Unit } func (p *TimeType) IsSetUnit() bool { - return p.Unit != nil + return p.Unit != nil } func (p *TimeType) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - var issetIsAdjustedToUTC bool = false; - var issetUnit bool = false; - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.BOOL { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - issetIsAdjustedToUTC = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - issetUnit = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - if !issetIsAdjustedToUTC{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field IsAdjustedToUTC is not set")); - } - if !issetUnit{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Unit is not set")); - } - return nil -} - -func (p *TimeType) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadBool(ctx); err != nil { - return thrift.PrependError("error reading field 1: ", err) -} else { - p.IsAdjustedToUTC = v -} - return nil -} - -func (p *TimeType) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - p.Unit = &TimeUnit{} - if err := p.Unit.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.Unit), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + var issetIsAdjustedToUTC bool = false + var issetUnit bool = false + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.BOOL { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + issetIsAdjustedToUTC = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + issetUnit = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + if !issetIsAdjustedToUTC { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field IsAdjustedToUTC is not set")) + } + if !issetUnit { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Unit is not set")) + } + return nil +} + +func (p *TimeType) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadBool(ctx); err != nil { + return thrift.PrependError("error reading field 1: ", err) + } else { + p.IsAdjustedToUTC = v + } + return nil +} + +func (p *TimeType) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + p.Unit = &TimeUnit{} + if err := p.Unit.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.Unit), err) + } + return nil } func (p *TimeType) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "TimeType"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "TimeType"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *TimeType) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "isAdjustedToUTC", thrift.BOOL, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:isAdjustedToUTC: ", p), err) } - if err := oprot.WriteBool(ctx, bool(p.IsAdjustedToUTC)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.isAdjustedToUTC (1) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:isAdjustedToUTC: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "isAdjustedToUTC", thrift.BOOL, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:isAdjustedToUTC: ", p), err) + } + if err := oprot.WriteBool(ctx, bool(p.IsAdjustedToUTC)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.isAdjustedToUTC (1) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:isAdjustedToUTC: ", p), err) + } + return err } func (p *TimeType) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "unit", thrift.STRUCT, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:unit: ", p), err) } - if err := p.Unit.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.Unit), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:unit: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "unit", thrift.STRUCT, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:unit: ", p), err) + } + if err := p.Unit.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.Unit), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:unit: ", p), err) + } + return err } func (p *TimeType) Equals(other *TimeType) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if p.IsAdjustedToUTC != other.IsAdjustedToUTC { return false } - if !p.Unit.Equals(other.Unit) { return false } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if p.IsAdjustedToUTC != other.IsAdjustedToUTC { + return false + } + if !p.Unit.Equals(other.Unit) { + return false + } + return true } func (p *TimeType) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("TimeType(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("TimeType(%+v)", *p) } func (p *TimeType) Validate() error { - return nil + return nil } + // Integer logical type annotation -// +// // bitWidth must be 8, 16, 32, or 64. -// +// // Allowed for physical types: INT32, INT64 -// +// // Attributes: -// - BitWidth -// - IsSigned +// - BitWidth +// - IsSigned type IntType struct { - BitWidth int8 `thrift:"bitWidth,1,required" db:"bitWidth" json:"bitWidth"` - IsSigned bool `thrift:"isSigned,2,required" db:"isSigned" json:"isSigned"` + BitWidth int8 `thrift:"bitWidth,1,required" db:"bitWidth" json:"bitWidth"` + IsSigned bool `thrift:"isSigned,2,required" db:"isSigned" json:"isSigned"` } func NewIntType() *IntType { - return &IntType{} + return &IntType{} } - func (p *IntType) GetBitWidth() int8 { - return p.BitWidth + return p.BitWidth } func (p *IntType) GetIsSigned() bool { - return p.IsSigned + return p.IsSigned } func (p *IntType) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - var issetBitWidth bool = false; - var issetIsSigned bool = false; - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.BYTE { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - issetBitWidth = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.BOOL { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - issetIsSigned = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - if !issetBitWidth{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field BitWidth is not set")); - } - if !issetIsSigned{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field IsSigned is not set")); - } - return nil -} - -func (p *IntType) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadByte(ctx); err != nil { - return thrift.PrependError("error reading field 1: ", err) -} else { - temp := int8(v) - p.BitWidth = temp -} - return nil -} - -func (p *IntType) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadBool(ctx); err != nil { - return thrift.PrependError("error reading field 2: ", err) -} else { - p.IsSigned = v -} - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + var issetBitWidth bool = false + var issetIsSigned bool = false + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.BYTE { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + issetBitWidth = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.BOOL { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + issetIsSigned = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + if !issetBitWidth { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field BitWidth is not set")) + } + if !issetIsSigned { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field IsSigned is not set")) + } + return nil +} + +func (p *IntType) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadByte(ctx); err != nil { + return thrift.PrependError("error reading field 1: ", err) + } else { + temp := int8(v) + p.BitWidth = temp + } + return nil +} + +func (p *IntType) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadBool(ctx); err != nil { + return thrift.PrependError("error reading field 2: ", err) + } else { + p.IsSigned = v + } + return nil } func (p *IntType) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "IntType"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "IntType"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *IntType) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "bitWidth", thrift.BYTE, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:bitWidth: ", p), err) } - if err := oprot.WriteByte(ctx, int8(p.BitWidth)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.bitWidth (1) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:bitWidth: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "bitWidth", thrift.BYTE, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:bitWidth: ", p), err) + } + if err := oprot.WriteByte(ctx, int8(p.BitWidth)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.bitWidth (1) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:bitWidth: ", p), err) + } + return err } func (p *IntType) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "isSigned", thrift.BOOL, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:isSigned: ", p), err) } - if err := oprot.WriteBool(ctx, bool(p.IsSigned)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.isSigned (2) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:isSigned: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "isSigned", thrift.BOOL, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:isSigned: ", p), err) + } + if err := oprot.WriteBool(ctx, bool(p.IsSigned)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.isSigned (2) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:isSigned: ", p), err) + } + return err } func (p *IntType) Equals(other *IntType) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if p.BitWidth != other.BitWidth { return false } - if p.IsSigned != other.IsSigned { return false } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if p.BitWidth != other.BitWidth { + return false + } + if p.IsSigned != other.IsSigned { + return false + } + return true } func (p *IntType) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("IntType(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("IntType(%+v)", *p) } func (p *IntType) Validate() error { - return nil + return nil } + // Embedded JSON logical type annotation -// +// // Allowed for physical types: BINARY type JsonType struct { } func NewJsonType() *JsonType { - return &JsonType{} + return &JsonType{} } func (p *JsonType) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil } func (p *JsonType) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "JsonType"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "JsonType"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *JsonType) Equals(other *JsonType) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + return true } func (p *JsonType) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("JsonType(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("JsonType(%+v)", *p) } func (p *JsonType) Validate() error { - return nil + return nil } + // Embedded BSON logical type annotation -// +// // Allowed for physical types: BINARY type BsonType struct { } func NewBsonType() *BsonType { - return &BsonType{} + return &BsonType{} } func (p *BsonType) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil } func (p *BsonType) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "BsonType"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "BsonType"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *BsonType) Equals(other *BsonType) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + return true } func (p *BsonType) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("BsonType(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("BsonType(%+v)", *p) } func (p *BsonType) Validate() error { - return nil + return nil } + // LogicalType annotations to replace ConvertedType. -// +// // To maintain compatibility, implementations using LogicalType for a // SchemaElement must also set the corresponding ConvertedType (if any) // from the following table. -// +// // Attributes: -// - STRING -// - MAP -// - LIST -// - ENUM -// - DECIMAL -// - DATE -// - TIME -// - TIMESTAMP -// - INTEGER -// - UNKNOWN -// - JSON -// - BSON -// - UUID -// - FLOAT16 +// - STRING +// - MAP +// - LIST +// - ENUM +// - DECIMAL +// - DATE +// - TIME +// - TIMESTAMP +// - INTEGER +// - UNKNOWN +// - JSON +// - BSON +// - UUID +// - FLOAT16 type LogicalType struct { - STRING *StringType `thrift:"STRING,1" db:"STRING" json:"STRING,omitempty"` - MAP *MapType `thrift:"MAP,2" db:"MAP" json:"MAP,omitempty"` - LIST *ListType `thrift:"LIST,3" db:"LIST" json:"LIST,omitempty"` - ENUM *EnumType `thrift:"ENUM,4" db:"ENUM" json:"ENUM,omitempty"` - DECIMAL *DecimalType `thrift:"DECIMAL,5" db:"DECIMAL" json:"DECIMAL,omitempty"` - DATE *DateType `thrift:"DATE,6" db:"DATE" json:"DATE,omitempty"` - TIME *TimeType `thrift:"TIME,7" db:"TIME" json:"TIME,omitempty"` - TIMESTAMP *TimestampType `thrift:"TIMESTAMP,8" db:"TIMESTAMP" json:"TIMESTAMP,omitempty"` - // unused field # 9 - INTEGER *IntType `thrift:"INTEGER,10" db:"INTEGER" json:"INTEGER,omitempty"` - UNKNOWN *NullType `thrift:"UNKNOWN,11" db:"UNKNOWN" json:"UNKNOWN,omitempty"` - JSON *JsonType `thrift:"JSON,12" db:"JSON" json:"JSON,omitempty"` - BSON *BsonType `thrift:"BSON,13" db:"BSON" json:"BSON,omitempty"` - UUID *UUIDType `thrift:"UUID,14" db:"UUID" json:"UUID,omitempty"` - FLOAT16 *Float16Type `thrift:"FLOAT16,15" db:"FLOAT16" json:"FLOAT16,omitempty"` + STRING *StringType `thrift:"STRING,1" db:"STRING" json:"STRING,omitempty"` + MAP *MapType `thrift:"MAP,2" db:"MAP" json:"MAP,omitempty"` + LIST *ListType `thrift:"LIST,3" db:"LIST" json:"LIST,omitempty"` + ENUM *EnumType `thrift:"ENUM,4" db:"ENUM" json:"ENUM,omitempty"` + DECIMAL *DecimalType `thrift:"DECIMAL,5" db:"DECIMAL" json:"DECIMAL,omitempty"` + DATE *DateType `thrift:"DATE,6" db:"DATE" json:"DATE,omitempty"` + TIME *TimeType `thrift:"TIME,7" db:"TIME" json:"TIME,omitempty"` + TIMESTAMP *TimestampType `thrift:"TIMESTAMP,8" db:"TIMESTAMP" json:"TIMESTAMP,omitempty"` + // unused field # 9 + INTEGER *IntType `thrift:"INTEGER,10" db:"INTEGER" json:"INTEGER,omitempty"` + UNKNOWN *NullType `thrift:"UNKNOWN,11" db:"UNKNOWN" json:"UNKNOWN,omitempty"` + JSON *JsonType `thrift:"JSON,12" db:"JSON" json:"JSON,omitempty"` + BSON *BsonType `thrift:"BSON,13" db:"BSON" json:"BSON,omitempty"` + UUID *UUIDType `thrift:"UUID,14" db:"UUID" json:"UUID,omitempty"` + FLOAT16 *Float16Type `thrift:"FLOAT16,15" db:"FLOAT16" json:"FLOAT16,omitempty"` } func NewLogicalType() *LogicalType { - return &LogicalType{} + return &LogicalType{} } var LogicalType_STRING_DEFAULT *StringType + func (p *LogicalType) GetSTRING() *StringType { - if !p.IsSetSTRING() { - return LogicalType_STRING_DEFAULT - } -return p.STRING + if !p.IsSetSTRING() { + return LogicalType_STRING_DEFAULT + } + return p.STRING } + var LogicalType_MAP_DEFAULT *MapType + func (p *LogicalType) GetMAP() *MapType { - if !p.IsSetMAP() { - return LogicalType_MAP_DEFAULT - } -return p.MAP + if !p.IsSetMAP() { + return LogicalType_MAP_DEFAULT + } + return p.MAP } + var LogicalType_LIST_DEFAULT *ListType + func (p *LogicalType) GetLIST() *ListType { - if !p.IsSetLIST() { - return LogicalType_LIST_DEFAULT - } -return p.LIST + if !p.IsSetLIST() { + return LogicalType_LIST_DEFAULT + } + return p.LIST } + var LogicalType_ENUM_DEFAULT *EnumType + func (p *LogicalType) GetENUM() *EnumType { - if !p.IsSetENUM() { - return LogicalType_ENUM_DEFAULT - } -return p.ENUM + if !p.IsSetENUM() { + return LogicalType_ENUM_DEFAULT + } + return p.ENUM } + var LogicalType_DECIMAL_DEFAULT *DecimalType + func (p *LogicalType) GetDECIMAL() *DecimalType { - if !p.IsSetDECIMAL() { - return LogicalType_DECIMAL_DEFAULT - } -return p.DECIMAL + if !p.IsSetDECIMAL() { + return LogicalType_DECIMAL_DEFAULT + } + return p.DECIMAL } + var LogicalType_DATE_DEFAULT *DateType + func (p *LogicalType) GetDATE() *DateType { - if !p.IsSetDATE() { - return LogicalType_DATE_DEFAULT - } -return p.DATE + if !p.IsSetDATE() { + return LogicalType_DATE_DEFAULT + } + return p.DATE } + var LogicalType_TIME_DEFAULT *TimeType + func (p *LogicalType) GetTIME() *TimeType { - if !p.IsSetTIME() { - return LogicalType_TIME_DEFAULT - } -return p.TIME + if !p.IsSetTIME() { + return LogicalType_TIME_DEFAULT + } + return p.TIME } + var LogicalType_TIMESTAMP_DEFAULT *TimestampType + func (p *LogicalType) GetTIMESTAMP() *TimestampType { - if !p.IsSetTIMESTAMP() { - return LogicalType_TIMESTAMP_DEFAULT - } -return p.TIMESTAMP + if !p.IsSetTIMESTAMP() { + return LogicalType_TIMESTAMP_DEFAULT + } + return p.TIMESTAMP } + var LogicalType_INTEGER_DEFAULT *IntType + func (p *LogicalType) GetINTEGER() *IntType { - if !p.IsSetINTEGER() { - return LogicalType_INTEGER_DEFAULT - } -return p.INTEGER + if !p.IsSetINTEGER() { + return LogicalType_INTEGER_DEFAULT + } + return p.INTEGER } + var LogicalType_UNKNOWN_DEFAULT *NullType + func (p *LogicalType) GetUNKNOWN() *NullType { - if !p.IsSetUNKNOWN() { - return LogicalType_UNKNOWN_DEFAULT - } -return p.UNKNOWN + if !p.IsSetUNKNOWN() { + return LogicalType_UNKNOWN_DEFAULT + } + return p.UNKNOWN } + var LogicalType_JSON_DEFAULT *JsonType + func (p *LogicalType) GetJSON() *JsonType { - if !p.IsSetJSON() { - return LogicalType_JSON_DEFAULT - } -return p.JSON + if !p.IsSetJSON() { + return LogicalType_JSON_DEFAULT + } + return p.JSON } + var LogicalType_BSON_DEFAULT *BsonType + func (p *LogicalType) GetBSON() *BsonType { - if !p.IsSetBSON() { - return LogicalType_BSON_DEFAULT - } -return p.BSON + if !p.IsSetBSON() { + return LogicalType_BSON_DEFAULT + } + return p.BSON } + var LogicalType_UUID_DEFAULT *UUIDType + func (p *LogicalType) GetUUID() *UUIDType { - if !p.IsSetUUID() { - return LogicalType_UUID_DEFAULT - } -return p.UUID + if !p.IsSetUUID() { + return LogicalType_UUID_DEFAULT + } + return p.UUID } + var LogicalType_FLOAT16_DEFAULT *Float16Type + func (p *LogicalType) GetFLOAT16() *Float16Type { - if !p.IsSetFLOAT16() { - return LogicalType_FLOAT16_DEFAULT - } -return p.FLOAT16 + if !p.IsSetFLOAT16() { + return LogicalType_FLOAT16_DEFAULT + } + return p.FLOAT16 } func (p *LogicalType) CountSetFieldsLogicalType() int { - count := 0 - if (p.IsSetSTRING()) { - count++ - } - if (p.IsSetMAP()) { - count++ - } - if (p.IsSetLIST()) { - count++ - } - if (p.IsSetENUM()) { - count++ - } - if (p.IsSetDECIMAL()) { - count++ - } - if (p.IsSetDATE()) { - count++ - } - if (p.IsSetTIME()) { - count++ - } - if (p.IsSetTIMESTAMP()) { - count++ - } - if (p.IsSetINTEGER()) { - count++ - } - if (p.IsSetUNKNOWN()) { - count++ - } - if (p.IsSetJSON()) { - count++ - } - if (p.IsSetBSON()) { - count++ - } - if (p.IsSetUUID()) { - count++ - } - if (p.IsSetFLOAT16()) { - count++ - } - return count + count := 0 + if p.IsSetSTRING() { + count++ + } + if p.IsSetMAP() { + count++ + } + if p.IsSetLIST() { + count++ + } + if p.IsSetENUM() { + count++ + } + if p.IsSetDECIMAL() { + count++ + } + if p.IsSetDATE() { + count++ + } + if p.IsSetTIME() { + count++ + } + if p.IsSetTIMESTAMP() { + count++ + } + if p.IsSetINTEGER() { + count++ + } + if p.IsSetUNKNOWN() { + count++ + } + if p.IsSetJSON() { + count++ + } + if p.IsSetBSON() { + count++ + } + if p.IsSetUUID() { + count++ + } + if p.IsSetFLOAT16() { + count++ + } + return count } func (p *LogicalType) IsSetSTRING() bool { - return p.STRING != nil + return p.STRING != nil } func (p *LogicalType) IsSetMAP() bool { - return p.MAP != nil + return p.MAP != nil } func (p *LogicalType) IsSetLIST() bool { - return p.LIST != nil + return p.LIST != nil } func (p *LogicalType) IsSetENUM() bool { - return p.ENUM != nil + return p.ENUM != nil } func (p *LogicalType) IsSetDECIMAL() bool { - return p.DECIMAL != nil + return p.DECIMAL != nil } func (p *LogicalType) IsSetDATE() bool { - return p.DATE != nil + return p.DATE != nil } func (p *LogicalType) IsSetTIME() bool { - return p.TIME != nil + return p.TIME != nil } func (p *LogicalType) IsSetTIMESTAMP() bool { - return p.TIMESTAMP != nil + return p.TIMESTAMP != nil } func (p *LogicalType) IsSetINTEGER() bool { - return p.INTEGER != nil + return p.INTEGER != nil } func (p *LogicalType) IsSetUNKNOWN() bool { - return p.UNKNOWN != nil + return p.UNKNOWN != nil } func (p *LogicalType) IsSetJSON() bool { - return p.JSON != nil + return p.JSON != nil } func (p *LogicalType) IsSetBSON() bool { - return p.BSON != nil + return p.BSON != nil } func (p *LogicalType) IsSetUUID() bool { - return p.UUID != nil + return p.UUID != nil } func (p *LogicalType) IsSetFLOAT16() bool { - return p.FLOAT16 != nil + return p.FLOAT16 != nil } func (p *LogicalType) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 3: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField3(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 4: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField4(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 5: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField5(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 6: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField6(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 7: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField7(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 8: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField8(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 10: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField10(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 11: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField11(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 12: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField12(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 13: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField13(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 14: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField14(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 15: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField15(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil -} - -func (p *LogicalType) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - p.STRING = &StringType{} - if err := p.STRING.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.STRING), err) - } - return nil -} - -func (p *LogicalType) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - p.MAP = &MapType{} - if err := p.MAP.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.MAP), err) - } - return nil -} - -func (p *LogicalType) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { - p.LIST = &ListType{} - if err := p.LIST.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.LIST), err) - } - return nil -} - -func (p *LogicalType) ReadField4(ctx context.Context, iprot thrift.TProtocol) error { - p.ENUM = &EnumType{} - if err := p.ENUM.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.ENUM), err) - } - return nil -} - -func (p *LogicalType) ReadField5(ctx context.Context, iprot thrift.TProtocol) error { - p.DECIMAL = &DecimalType{} - if err := p.DECIMAL.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.DECIMAL), err) - } - return nil -} - -func (p *LogicalType) ReadField6(ctx context.Context, iprot thrift.TProtocol) error { - p.DATE = &DateType{} - if err := p.DATE.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.DATE), err) - } - return nil -} - -func (p *LogicalType) ReadField7(ctx context.Context, iprot thrift.TProtocol) error { - p.TIME = &TimeType{} - if err := p.TIME.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.TIME), err) - } - return nil -} - -func (p *LogicalType) ReadField8(ctx context.Context, iprot thrift.TProtocol) error { - p.TIMESTAMP = &TimestampType{} - if err := p.TIMESTAMP.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.TIMESTAMP), err) - } - return nil -} - -func (p *LogicalType) ReadField10(ctx context.Context, iprot thrift.TProtocol) error { - p.INTEGER = &IntType{} - if err := p.INTEGER.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.INTEGER), err) - } - return nil -} - -func (p *LogicalType) ReadField11(ctx context.Context, iprot thrift.TProtocol) error { - p.UNKNOWN = &NullType{} - if err := p.UNKNOWN.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.UNKNOWN), err) - } - return nil -} - -func (p *LogicalType) ReadField12(ctx context.Context, iprot thrift.TProtocol) error { - p.JSON = &JsonType{} - if err := p.JSON.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.JSON), err) - } - return nil -} - -func (p *LogicalType) ReadField13(ctx context.Context, iprot thrift.TProtocol) error { - p.BSON = &BsonType{} - if err := p.BSON.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.BSON), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 3: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField3(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 4: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField4(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 5: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField5(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 6: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField6(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 7: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField7(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 8: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField8(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 10: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField10(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 11: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField11(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 12: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField12(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 13: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField13(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 14: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField14(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 15: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField15(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil +} + +func (p *LogicalType) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + p.STRING = &StringType{} + if err := p.STRING.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.STRING), err) + } + return nil +} + +func (p *LogicalType) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + p.MAP = &MapType{} + if err := p.MAP.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.MAP), err) + } + return nil +} + +func (p *LogicalType) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { + p.LIST = &ListType{} + if err := p.LIST.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.LIST), err) + } + return nil +} + +func (p *LogicalType) ReadField4(ctx context.Context, iprot thrift.TProtocol) error { + p.ENUM = &EnumType{} + if err := p.ENUM.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.ENUM), err) + } + return nil +} + +func (p *LogicalType) ReadField5(ctx context.Context, iprot thrift.TProtocol) error { + p.DECIMAL = &DecimalType{} + if err := p.DECIMAL.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.DECIMAL), err) + } + return nil +} + +func (p *LogicalType) ReadField6(ctx context.Context, iprot thrift.TProtocol) error { + p.DATE = &DateType{} + if err := p.DATE.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.DATE), err) + } + return nil +} + +func (p *LogicalType) ReadField7(ctx context.Context, iprot thrift.TProtocol) error { + p.TIME = &TimeType{} + if err := p.TIME.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.TIME), err) + } + return nil +} + +func (p *LogicalType) ReadField8(ctx context.Context, iprot thrift.TProtocol) error { + p.TIMESTAMP = &TimestampType{} + if err := p.TIMESTAMP.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.TIMESTAMP), err) + } + return nil +} + +func (p *LogicalType) ReadField10(ctx context.Context, iprot thrift.TProtocol) error { + p.INTEGER = &IntType{} + if err := p.INTEGER.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.INTEGER), err) + } + return nil +} + +func (p *LogicalType) ReadField11(ctx context.Context, iprot thrift.TProtocol) error { + p.UNKNOWN = &NullType{} + if err := p.UNKNOWN.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.UNKNOWN), err) + } + return nil +} + +func (p *LogicalType) ReadField12(ctx context.Context, iprot thrift.TProtocol) error { + p.JSON = &JsonType{} + if err := p.JSON.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.JSON), err) + } + return nil +} + +func (p *LogicalType) ReadField13(ctx context.Context, iprot thrift.TProtocol) error { + p.BSON = &BsonType{} + if err := p.BSON.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.BSON), err) + } + return nil } -func (p *LogicalType) ReadField14(ctx context.Context, iprot thrift.TProtocol) error { - p.UUID = &UUIDType{} - if err := p.UUID.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.UUID), err) - } - return nil +func (p *LogicalType) ReadField14(ctx context.Context, iprot thrift.TProtocol) error { + p.UUID = &UUIDType{} + if err := p.UUID.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.UUID), err) + } + return nil } -func (p *LogicalType) ReadField15(ctx context.Context, iprot thrift.TProtocol) error { - p.FLOAT16 = &Float16Type{} - if err := p.FLOAT16.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.FLOAT16), err) - } - return nil +func (p *LogicalType) ReadField15(ctx context.Context, iprot thrift.TProtocol) error { + p.FLOAT16 = &Float16Type{} + if err := p.FLOAT16.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.FLOAT16), err) + } + return nil } func (p *LogicalType) Write(ctx context.Context, oprot thrift.TProtocol) error { - if c := p.CountSetFieldsLogicalType(); c != 1 { - return fmt.Errorf("%T write union: exactly one field must be set (%d set)", p, c) - } - if err := oprot.WriteStructBegin(ctx, "LogicalType"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - if err := p.writeField3(ctx, oprot); err != nil { return err } - if err := p.writeField4(ctx, oprot); err != nil { return err } - if err := p.writeField5(ctx, oprot); err != nil { return err } - if err := p.writeField6(ctx, oprot); err != nil { return err } - if err := p.writeField7(ctx, oprot); err != nil { return err } - if err := p.writeField8(ctx, oprot); err != nil { return err } - if err := p.writeField10(ctx, oprot); err != nil { return err } - if err := p.writeField11(ctx, oprot); err != nil { return err } - if err := p.writeField12(ctx, oprot); err != nil { return err } - if err := p.writeField13(ctx, oprot); err != nil { return err } - if err := p.writeField14(ctx, oprot); err != nil { return err } - if err := p.writeField15(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if c := p.CountSetFieldsLogicalType(); c != 1 { + return fmt.Errorf("%T write union: exactly one field must be set (%d set)", p, c) + } + if err := oprot.WriteStructBegin(ctx, "LogicalType"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + if err := p.writeField3(ctx, oprot); err != nil { + return err + } + if err := p.writeField4(ctx, oprot); err != nil { + return err + } + if err := p.writeField5(ctx, oprot); err != nil { + return err + } + if err := p.writeField6(ctx, oprot); err != nil { + return err + } + if err := p.writeField7(ctx, oprot); err != nil { + return err + } + if err := p.writeField8(ctx, oprot); err != nil { + return err + } + if err := p.writeField10(ctx, oprot); err != nil { + return err + } + if err := p.writeField11(ctx, oprot); err != nil { + return err + } + if err := p.writeField12(ctx, oprot); err != nil { + return err + } + if err := p.writeField13(ctx, oprot); err != nil { + return err + } + if err := p.writeField14(ctx, oprot); err != nil { + return err + } + if err := p.writeField15(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *LogicalType) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetSTRING() { - if err := oprot.WriteFieldBegin(ctx, "STRING", thrift.STRUCT, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:STRING: ", p), err) } - if err := p.STRING.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.STRING), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:STRING: ", p), err) } - } - return err + if p.IsSetSTRING() { + if err := oprot.WriteFieldBegin(ctx, "STRING", thrift.STRUCT, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:STRING: ", p), err) + } + if err := p.STRING.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.STRING), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:STRING: ", p), err) + } + } + return err } func (p *LogicalType) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetMAP() { - if err := oprot.WriteFieldBegin(ctx, "MAP", thrift.STRUCT, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:MAP: ", p), err) } - if err := p.MAP.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.MAP), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:MAP: ", p), err) } - } - return err + if p.IsSetMAP() { + if err := oprot.WriteFieldBegin(ctx, "MAP", thrift.STRUCT, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:MAP: ", p), err) + } + if err := p.MAP.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.MAP), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:MAP: ", p), err) + } + } + return err } func (p *LogicalType) writeField3(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetLIST() { - if err := oprot.WriteFieldBegin(ctx, "LIST", thrift.STRUCT, 3); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:LIST: ", p), err) } - if err := p.LIST.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.LIST), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 3:LIST: ", p), err) } - } - return err + if p.IsSetLIST() { + if err := oprot.WriteFieldBegin(ctx, "LIST", thrift.STRUCT, 3); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:LIST: ", p), err) + } + if err := p.LIST.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.LIST), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 3:LIST: ", p), err) + } + } + return err } func (p *LogicalType) writeField4(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetENUM() { - if err := oprot.WriteFieldBegin(ctx, "ENUM", thrift.STRUCT, 4); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:ENUM: ", p), err) } - if err := p.ENUM.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.ENUM), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 4:ENUM: ", p), err) } - } - return err + if p.IsSetENUM() { + if err := oprot.WriteFieldBegin(ctx, "ENUM", thrift.STRUCT, 4); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:ENUM: ", p), err) + } + if err := p.ENUM.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.ENUM), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 4:ENUM: ", p), err) + } + } + return err } func (p *LogicalType) writeField5(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetDECIMAL() { - if err := oprot.WriteFieldBegin(ctx, "DECIMAL", thrift.STRUCT, 5); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:DECIMAL: ", p), err) } - if err := p.DECIMAL.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.DECIMAL), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 5:DECIMAL: ", p), err) } - } - return err + if p.IsSetDECIMAL() { + if err := oprot.WriteFieldBegin(ctx, "DECIMAL", thrift.STRUCT, 5); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:DECIMAL: ", p), err) + } + if err := p.DECIMAL.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.DECIMAL), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 5:DECIMAL: ", p), err) + } + } + return err } func (p *LogicalType) writeField6(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetDATE() { - if err := oprot.WriteFieldBegin(ctx, "DATE", thrift.STRUCT, 6); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 6:DATE: ", p), err) } - if err := p.DATE.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.DATE), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 6:DATE: ", p), err) } - } - return err + if p.IsSetDATE() { + if err := oprot.WriteFieldBegin(ctx, "DATE", thrift.STRUCT, 6); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 6:DATE: ", p), err) + } + if err := p.DATE.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.DATE), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 6:DATE: ", p), err) + } + } + return err } func (p *LogicalType) writeField7(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetTIME() { - if err := oprot.WriteFieldBegin(ctx, "TIME", thrift.STRUCT, 7); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 7:TIME: ", p), err) } - if err := p.TIME.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.TIME), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 7:TIME: ", p), err) } - } - return err + if p.IsSetTIME() { + if err := oprot.WriteFieldBegin(ctx, "TIME", thrift.STRUCT, 7); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 7:TIME: ", p), err) + } + if err := p.TIME.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.TIME), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 7:TIME: ", p), err) + } + } + return err } func (p *LogicalType) writeField8(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetTIMESTAMP() { - if err := oprot.WriteFieldBegin(ctx, "TIMESTAMP", thrift.STRUCT, 8); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 8:TIMESTAMP: ", p), err) } - if err := p.TIMESTAMP.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.TIMESTAMP), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 8:TIMESTAMP: ", p), err) } - } - return err + if p.IsSetTIMESTAMP() { + if err := oprot.WriteFieldBegin(ctx, "TIMESTAMP", thrift.STRUCT, 8); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 8:TIMESTAMP: ", p), err) + } + if err := p.TIMESTAMP.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.TIMESTAMP), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 8:TIMESTAMP: ", p), err) + } + } + return err } func (p *LogicalType) writeField10(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetINTEGER() { - if err := oprot.WriteFieldBegin(ctx, "INTEGER", thrift.STRUCT, 10); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 10:INTEGER: ", p), err) } - if err := p.INTEGER.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.INTEGER), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 10:INTEGER: ", p), err) } - } - return err + if p.IsSetINTEGER() { + if err := oprot.WriteFieldBegin(ctx, "INTEGER", thrift.STRUCT, 10); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 10:INTEGER: ", p), err) + } + if err := p.INTEGER.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.INTEGER), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 10:INTEGER: ", p), err) + } + } + return err } func (p *LogicalType) writeField11(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetUNKNOWN() { - if err := oprot.WriteFieldBegin(ctx, "UNKNOWN", thrift.STRUCT, 11); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 11:UNKNOWN: ", p), err) } - if err := p.UNKNOWN.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.UNKNOWN), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 11:UNKNOWN: ", p), err) } - } - return err + if p.IsSetUNKNOWN() { + if err := oprot.WriteFieldBegin(ctx, "UNKNOWN", thrift.STRUCT, 11); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 11:UNKNOWN: ", p), err) + } + if err := p.UNKNOWN.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.UNKNOWN), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 11:UNKNOWN: ", p), err) + } + } + return err } func (p *LogicalType) writeField12(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetJSON() { - if err := oprot.WriteFieldBegin(ctx, "JSON", thrift.STRUCT, 12); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 12:JSON: ", p), err) } - if err := p.JSON.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.JSON), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 12:JSON: ", p), err) } - } - return err + if p.IsSetJSON() { + if err := oprot.WriteFieldBegin(ctx, "JSON", thrift.STRUCT, 12); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 12:JSON: ", p), err) + } + if err := p.JSON.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.JSON), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 12:JSON: ", p), err) + } + } + return err } func (p *LogicalType) writeField13(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetBSON() { - if err := oprot.WriteFieldBegin(ctx, "BSON", thrift.STRUCT, 13); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 13:BSON: ", p), err) } - if err := p.BSON.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.BSON), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 13:BSON: ", p), err) } - } - return err + if p.IsSetBSON() { + if err := oprot.WriteFieldBegin(ctx, "BSON", thrift.STRUCT, 13); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 13:BSON: ", p), err) + } + if err := p.BSON.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.BSON), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 13:BSON: ", p), err) + } + } + return err } func (p *LogicalType) writeField14(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetUUID() { - if err := oprot.WriteFieldBegin(ctx, "UUID", thrift.STRUCT, 14); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 14:UUID: ", p), err) } - if err := p.UUID.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.UUID), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 14:UUID: ", p), err) } - } - return err + if p.IsSetUUID() { + if err := oprot.WriteFieldBegin(ctx, "UUID", thrift.STRUCT, 14); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 14:UUID: ", p), err) + } + if err := p.UUID.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.UUID), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 14:UUID: ", p), err) + } + } + return err } func (p *LogicalType) writeField15(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetFLOAT16() { - if err := oprot.WriteFieldBegin(ctx, "FLOAT16", thrift.STRUCT, 15); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 15:FLOAT16: ", p), err) } - if err := p.FLOAT16.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.FLOAT16), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 15:FLOAT16: ", p), err) } - } - return err + if p.IsSetFLOAT16() { + if err := oprot.WriteFieldBegin(ctx, "FLOAT16", thrift.STRUCT, 15); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 15:FLOAT16: ", p), err) + } + if err := p.FLOAT16.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.FLOAT16), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 15:FLOAT16: ", p), err) + } + } + return err } func (p *LogicalType) Equals(other *LogicalType) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if !p.STRING.Equals(other.STRING) { return false } - if !p.MAP.Equals(other.MAP) { return false } - if !p.LIST.Equals(other.LIST) { return false } - if !p.ENUM.Equals(other.ENUM) { return false } - if !p.DECIMAL.Equals(other.DECIMAL) { return false } - if !p.DATE.Equals(other.DATE) { return false } - if !p.TIME.Equals(other.TIME) { return false } - if !p.TIMESTAMP.Equals(other.TIMESTAMP) { return false } - if !p.INTEGER.Equals(other.INTEGER) { return false } - if !p.UNKNOWN.Equals(other.UNKNOWN) { return false } - if !p.JSON.Equals(other.JSON) { return false } - if !p.BSON.Equals(other.BSON) { return false } - if !p.UUID.Equals(other.UUID) { return false } - if !p.FLOAT16.Equals(other.FLOAT16) { return false } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if !p.STRING.Equals(other.STRING) { + return false + } + if !p.MAP.Equals(other.MAP) { + return false + } + if !p.LIST.Equals(other.LIST) { + return false + } + if !p.ENUM.Equals(other.ENUM) { + return false + } + if !p.DECIMAL.Equals(other.DECIMAL) { + return false + } + if !p.DATE.Equals(other.DATE) { + return false + } + if !p.TIME.Equals(other.TIME) { + return false + } + if !p.TIMESTAMP.Equals(other.TIMESTAMP) { + return false + } + if !p.INTEGER.Equals(other.INTEGER) { + return false + } + if !p.UNKNOWN.Equals(other.UNKNOWN) { + return false + } + if !p.JSON.Equals(other.JSON) { + return false + } + if !p.BSON.Equals(other.BSON) { + return false + } + if !p.UUID.Equals(other.UUID) { + return false + } + if !p.FLOAT16.Equals(other.FLOAT16) { + return false + } + return true } func (p *LogicalType) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("LogicalType(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("LogicalType(%+v)", *p) } func (p *LogicalType) Validate() error { - return nil + return nil } + // Represents a element inside a schema definition. -// - if it is a group (inner node) then type is undefined and num_children is defined -// - if it is a primitive type (leaf) then type is defined and num_children is undefined +// - if it is a group (inner node) then type is undefined and num_children is defined +// - if it is a primitive type (leaf) then type is defined and num_children is undefined +// // the nodes are listed in depth first traversal order. -// +// // Attributes: -// - Type: Data type for this field. Not set if the current element is a non-leaf node -// - TypeLength: If type is FIXED_LEN_BYTE_ARRAY, this is the byte length of the values. +// - Type: Data type for this field. Not set if the current element is a non-leaf node +// - TypeLength: If type is FIXED_LEN_BYTE_ARRAY, this is the byte length of the values. +// // Otherwise, if specified, this is the maximum bit length to store any of the values. // (e.g. a low cardinality INT col could have this set to 3). Note that this is // in the schema, and therefore fixed for the entire file. -// - RepetitionType: repetition of the field. The root of the schema does not have a repetition_type. +// - RepetitionType: repetition of the field. The root of the schema does not have a repetition_type. +// // All other nodes must have one -// - Name: Name of the field in the schema -// - NumChildren: Nested fields. Since thrift does not support nested fields, +// - Name: Name of the field in the schema +// - NumChildren: Nested fields. Since thrift does not support nested fields, +// // the nesting is flattened to a single list by a depth-first traversal. // The children count is used to construct the nested relationship. // This field is not set when the element is a primitive type -// - ConvertedType: DEPRECATED: When the schema is the result of a conversion from another model. +// - ConvertedType: DEPRECATED: When the schema is the result of a conversion from another model. +// // Used to record the original type to help with cross conversion. -// +// // This is superseded by logicalType. -// - Scale: DEPRECATED: Used when this column contains decimal data. +// - Scale: DEPRECATED: Used when this column contains decimal data. +// // See the DECIMAL converted type for more details. -// +// // This is superseded by using the DecimalType annotation in logicalType. -// - Precision -// - FieldID: When the original schema supports field ids, this will save the +// - Precision +// - FieldID: When the original schema supports field ids, this will save the +// // original field id in the parquet schema -// - LogicalType: The logical type of this SchemaElement -// +// - LogicalType: The logical type of this SchemaElement +// // LogicalType replaces ConvertedType, but ConvertedType is still required // for some logical types to ensure forward-compatibility in format v1. type SchemaElement struct { - Type *Type `thrift:"type,1" db:"type" json:"type,omitempty"` - TypeLength *int32 `thrift:"type_length,2" db:"type_length" json:"type_length,omitempty"` - RepetitionType *FieldRepetitionType `thrift:"repetition_type,3" db:"repetition_type" json:"repetition_type,omitempty"` - Name string `thrift:"name,4,required" db:"name" json:"name"` - NumChildren *int32 `thrift:"num_children,5" db:"num_children" json:"num_children,omitempty"` - ConvertedType *ConvertedType `thrift:"converted_type,6" db:"converted_type" json:"converted_type,omitempty"` - Scale *int32 `thrift:"scale,7" db:"scale" json:"scale,omitempty"` - Precision *int32 `thrift:"precision,8" db:"precision" json:"precision,omitempty"` - FieldID *int32 `thrift:"field_id,9" db:"field_id" json:"field_id,omitempty"` - LogicalType *LogicalType `thrift:"logicalType,10" db:"logicalType" json:"logicalType,omitempty"` + Type *Type `thrift:"type,1" db:"type" json:"type,omitempty"` + TypeLength *int32 `thrift:"type_length,2" db:"type_length" json:"type_length,omitempty"` + RepetitionType *FieldRepetitionType `thrift:"repetition_type,3" db:"repetition_type" json:"repetition_type,omitempty"` + Name string `thrift:"name,4,required" db:"name" json:"name"` + NumChildren *int32 `thrift:"num_children,5" db:"num_children" json:"num_children,omitempty"` + ConvertedType *ConvertedType `thrift:"converted_type,6" db:"converted_type" json:"converted_type,omitempty"` + Scale *int32 `thrift:"scale,7" db:"scale" json:"scale,omitempty"` + Precision *int32 `thrift:"precision,8" db:"precision" json:"precision,omitempty"` + FieldID *int32 `thrift:"field_id,9" db:"field_id" json:"field_id,omitempty"` + LogicalType *LogicalType `thrift:"logicalType,10" db:"logicalType" json:"logicalType,omitempty"` } func NewSchemaElement() *SchemaElement { - return &SchemaElement{} + return &SchemaElement{} } var SchemaElement_Type_DEFAULT Type + func (p *SchemaElement) GetType() Type { - if !p.IsSetType() { - return SchemaElement_Type_DEFAULT - } -return *p.Type + if !p.IsSetType() { + return SchemaElement_Type_DEFAULT + } + return *p.Type } + var SchemaElement_TypeLength_DEFAULT int32 + func (p *SchemaElement) GetTypeLength() int32 { - if !p.IsSetTypeLength() { - return SchemaElement_TypeLength_DEFAULT - } -return *p.TypeLength + if !p.IsSetTypeLength() { + return SchemaElement_TypeLength_DEFAULT + } + return *p.TypeLength } + var SchemaElement_RepetitionType_DEFAULT FieldRepetitionType + func (p *SchemaElement) GetRepetitionType() FieldRepetitionType { - if !p.IsSetRepetitionType() { - return SchemaElement_RepetitionType_DEFAULT - } -return *p.RepetitionType + if !p.IsSetRepetitionType() { + return SchemaElement_RepetitionType_DEFAULT + } + return *p.RepetitionType } func (p *SchemaElement) GetName() string { - return p.Name + return p.Name } + var SchemaElement_NumChildren_DEFAULT int32 + func (p *SchemaElement) GetNumChildren() int32 { - if !p.IsSetNumChildren() { - return SchemaElement_NumChildren_DEFAULT - } -return *p.NumChildren + if !p.IsSetNumChildren() { + return SchemaElement_NumChildren_DEFAULT + } + return *p.NumChildren } + var SchemaElement_ConvertedType_DEFAULT ConvertedType + func (p *SchemaElement) GetConvertedType() ConvertedType { - if !p.IsSetConvertedType() { - return SchemaElement_ConvertedType_DEFAULT - } -return *p.ConvertedType + if !p.IsSetConvertedType() { + return SchemaElement_ConvertedType_DEFAULT + } + return *p.ConvertedType } + var SchemaElement_Scale_DEFAULT int32 + func (p *SchemaElement) GetScale() int32 { - if !p.IsSetScale() { - return SchemaElement_Scale_DEFAULT - } -return *p.Scale + if !p.IsSetScale() { + return SchemaElement_Scale_DEFAULT + } + return *p.Scale } + var SchemaElement_Precision_DEFAULT int32 + func (p *SchemaElement) GetPrecision() int32 { - if !p.IsSetPrecision() { - return SchemaElement_Precision_DEFAULT - } -return *p.Precision + if !p.IsSetPrecision() { + return SchemaElement_Precision_DEFAULT + } + return *p.Precision } + var SchemaElement_FieldID_DEFAULT int32 + func (p *SchemaElement) GetFieldID() int32 { - if !p.IsSetFieldID() { - return SchemaElement_FieldID_DEFAULT - } -return *p.FieldID + if !p.IsSetFieldID() { + return SchemaElement_FieldID_DEFAULT + } + return *p.FieldID } + var SchemaElement_LogicalType_DEFAULT *LogicalType + func (p *SchemaElement) GetLogicalType() *LogicalType { - if !p.IsSetLogicalType() { - return SchemaElement_LogicalType_DEFAULT - } -return p.LogicalType + if !p.IsSetLogicalType() { + return SchemaElement_LogicalType_DEFAULT + } + return p.LogicalType } func (p *SchemaElement) IsSetType() bool { - return p.Type != nil + return p.Type != nil } func (p *SchemaElement) IsSetTypeLength() bool { - return p.TypeLength != nil + return p.TypeLength != nil } func (p *SchemaElement) IsSetRepetitionType() bool { - return p.RepetitionType != nil + return p.RepetitionType != nil } func (p *SchemaElement) IsSetNumChildren() bool { - return p.NumChildren != nil + return p.NumChildren != nil } func (p *SchemaElement) IsSetConvertedType() bool { - return p.ConvertedType != nil + return p.ConvertedType != nil } func (p *SchemaElement) IsSetScale() bool { - return p.Scale != nil + return p.Scale != nil } func (p *SchemaElement) IsSetPrecision() bool { - return p.Precision != nil + return p.Precision != nil } func (p *SchemaElement) IsSetFieldID() bool { - return p.FieldID != nil + return p.FieldID != nil } func (p *SchemaElement) IsSetLogicalType() bool { - return p.LogicalType != nil + return p.LogicalType != nil } func (p *SchemaElement) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - var issetName bool = false; - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.I32 { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.I32 { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 3: - if fieldTypeId == thrift.I32 { - if err := p.ReadField3(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 4: - if fieldTypeId == thrift.STRING { - if err := p.ReadField4(ctx, iprot); err != nil { - return err - } - issetName = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 5: - if fieldTypeId == thrift.I32 { - if err := p.ReadField5(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 6: - if fieldTypeId == thrift.I32 { - if err := p.ReadField6(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 7: - if fieldTypeId == thrift.I32 { - if err := p.ReadField7(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 8: - if fieldTypeId == thrift.I32 { - if err := p.ReadField8(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 9: - if fieldTypeId == thrift.I32 { - if err := p.ReadField9(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 10: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField10(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - if !issetName{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Name is not set")); - } - return nil -} - -func (p *SchemaElement) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 1: ", err) -} else { - temp := Type(v) - p.Type = &temp -} - return nil -} - -func (p *SchemaElement) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 2: ", err) -} else { - p.TypeLength = &v -} - return nil -} - -func (p *SchemaElement) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 3: ", err) -} else { - temp := FieldRepetitionType(v) - p.RepetitionType = &temp -} - return nil -} - -func (p *SchemaElement) ReadField4(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadString(ctx); err != nil { - return thrift.PrependError("error reading field 4: ", err) -} else { - p.Name = v -} - return nil -} - -func (p *SchemaElement) ReadField5(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 5: ", err) -} else { - p.NumChildren = &v -} - return nil -} - -func (p *SchemaElement) ReadField6(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 6: ", err) -} else { - temp := ConvertedType(v) - p.ConvertedType = &temp -} - return nil -} - -func (p *SchemaElement) ReadField7(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 7: ", err) -} else { - p.Scale = &v -} - return nil -} - -func (p *SchemaElement) ReadField8(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 8: ", err) -} else { - p.Precision = &v -} - return nil -} - -func (p *SchemaElement) ReadField9(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 9: ", err) -} else { - p.FieldID = &v -} - return nil -} - -func (p *SchemaElement) ReadField10(ctx context.Context, iprot thrift.TProtocol) error { - p.LogicalType = &LogicalType{} - if err := p.LogicalType.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.LogicalType), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + var issetName bool = false + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.I32 { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.I32 { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 3: + if fieldTypeId == thrift.I32 { + if err := p.ReadField3(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 4: + if fieldTypeId == thrift.STRING { + if err := p.ReadField4(ctx, iprot); err != nil { + return err + } + issetName = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 5: + if fieldTypeId == thrift.I32 { + if err := p.ReadField5(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 6: + if fieldTypeId == thrift.I32 { + if err := p.ReadField6(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 7: + if fieldTypeId == thrift.I32 { + if err := p.ReadField7(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 8: + if fieldTypeId == thrift.I32 { + if err := p.ReadField8(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 9: + if fieldTypeId == thrift.I32 { + if err := p.ReadField9(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 10: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField10(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + if !issetName { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Name is not set")) + } + return nil +} + +func (p *SchemaElement) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 1: ", err) + } else { + temp := Type(v) + p.Type = &temp + } + return nil +} + +func (p *SchemaElement) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 2: ", err) + } else { + p.TypeLength = &v + } + return nil +} + +func (p *SchemaElement) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 3: ", err) + } else { + temp := FieldRepetitionType(v) + p.RepetitionType = &temp + } + return nil +} + +func (p *SchemaElement) ReadField4(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadString(ctx); err != nil { + return thrift.PrependError("error reading field 4: ", err) + } else { + p.Name = v + } + return nil +} + +func (p *SchemaElement) ReadField5(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 5: ", err) + } else { + p.NumChildren = &v + } + return nil +} + +func (p *SchemaElement) ReadField6(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 6: ", err) + } else { + temp := ConvertedType(v) + p.ConvertedType = &temp + } + return nil +} + +func (p *SchemaElement) ReadField7(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 7: ", err) + } else { + p.Scale = &v + } + return nil +} + +func (p *SchemaElement) ReadField8(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 8: ", err) + } else { + p.Precision = &v + } + return nil +} + +func (p *SchemaElement) ReadField9(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 9: ", err) + } else { + p.FieldID = &v + } + return nil +} + +func (p *SchemaElement) ReadField10(ctx context.Context, iprot thrift.TProtocol) error { + p.LogicalType = &LogicalType{} + if err := p.LogicalType.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.LogicalType), err) + } + return nil } func (p *SchemaElement) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "SchemaElement"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - if err := p.writeField3(ctx, oprot); err != nil { return err } - if err := p.writeField4(ctx, oprot); err != nil { return err } - if err := p.writeField5(ctx, oprot); err != nil { return err } - if err := p.writeField6(ctx, oprot); err != nil { return err } - if err := p.writeField7(ctx, oprot); err != nil { return err } - if err := p.writeField8(ctx, oprot); err != nil { return err } - if err := p.writeField9(ctx, oprot); err != nil { return err } - if err := p.writeField10(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "SchemaElement"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + if err := p.writeField3(ctx, oprot); err != nil { + return err + } + if err := p.writeField4(ctx, oprot); err != nil { + return err + } + if err := p.writeField5(ctx, oprot); err != nil { + return err + } + if err := p.writeField6(ctx, oprot); err != nil { + return err + } + if err := p.writeField7(ctx, oprot); err != nil { + return err + } + if err := p.writeField8(ctx, oprot); err != nil { + return err + } + if err := p.writeField9(ctx, oprot); err != nil { + return err + } + if err := p.writeField10(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *SchemaElement) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetType() { - if err := oprot.WriteFieldBegin(ctx, "type", thrift.I32, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:type: ", p), err) } - if err := oprot.WriteI32(ctx, int32(*p.Type)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.type (1) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:type: ", p), err) } - } - return err + if p.IsSetType() { + if err := oprot.WriteFieldBegin(ctx, "type", thrift.I32, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:type: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(*p.Type)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.type (1) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:type: ", p), err) + } + } + return err } func (p *SchemaElement) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetTypeLength() { - if err := oprot.WriteFieldBegin(ctx, "type_length", thrift.I32, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:type_length: ", p), err) } - if err := oprot.WriteI32(ctx, int32(*p.TypeLength)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.type_length (2) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:type_length: ", p), err) } - } - return err + if p.IsSetTypeLength() { + if err := oprot.WriteFieldBegin(ctx, "type_length", thrift.I32, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:type_length: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(*p.TypeLength)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.type_length (2) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:type_length: ", p), err) + } + } + return err } func (p *SchemaElement) writeField3(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetRepetitionType() { - if err := oprot.WriteFieldBegin(ctx, "repetition_type", thrift.I32, 3); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:repetition_type: ", p), err) } - if err := oprot.WriteI32(ctx, int32(*p.RepetitionType)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.repetition_type (3) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 3:repetition_type: ", p), err) } - } - return err + if p.IsSetRepetitionType() { + if err := oprot.WriteFieldBegin(ctx, "repetition_type", thrift.I32, 3); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:repetition_type: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(*p.RepetitionType)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.repetition_type (3) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 3:repetition_type: ", p), err) + } + } + return err } func (p *SchemaElement) writeField4(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "name", thrift.STRING, 4); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:name: ", p), err) } - if err := oprot.WriteString(ctx, string(p.Name)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.name (4) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 4:name: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "name", thrift.STRING, 4); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:name: ", p), err) + } + if err := oprot.WriteString(ctx, string(p.Name)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.name (4) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 4:name: ", p), err) + } + return err } func (p *SchemaElement) writeField5(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetNumChildren() { - if err := oprot.WriteFieldBegin(ctx, "num_children", thrift.I32, 5); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:num_children: ", p), err) } - if err := oprot.WriteI32(ctx, int32(*p.NumChildren)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.num_children (5) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 5:num_children: ", p), err) } - } - return err + if p.IsSetNumChildren() { + if err := oprot.WriteFieldBegin(ctx, "num_children", thrift.I32, 5); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:num_children: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(*p.NumChildren)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.num_children (5) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 5:num_children: ", p), err) + } + } + return err } func (p *SchemaElement) writeField6(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetConvertedType() { - if err := oprot.WriteFieldBegin(ctx, "converted_type", thrift.I32, 6); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 6:converted_type: ", p), err) } - if err := oprot.WriteI32(ctx, int32(*p.ConvertedType)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.converted_type (6) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 6:converted_type: ", p), err) } - } - return err + if p.IsSetConvertedType() { + if err := oprot.WriteFieldBegin(ctx, "converted_type", thrift.I32, 6); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 6:converted_type: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(*p.ConvertedType)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.converted_type (6) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 6:converted_type: ", p), err) + } + } + return err } func (p *SchemaElement) writeField7(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetScale() { - if err := oprot.WriteFieldBegin(ctx, "scale", thrift.I32, 7); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 7:scale: ", p), err) } - if err := oprot.WriteI32(ctx, int32(*p.Scale)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.scale (7) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 7:scale: ", p), err) } - } - return err + if p.IsSetScale() { + if err := oprot.WriteFieldBegin(ctx, "scale", thrift.I32, 7); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 7:scale: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(*p.Scale)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.scale (7) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 7:scale: ", p), err) + } + } + return err } func (p *SchemaElement) writeField8(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetPrecision() { - if err := oprot.WriteFieldBegin(ctx, "precision", thrift.I32, 8); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 8:precision: ", p), err) } - if err := oprot.WriteI32(ctx, int32(*p.Precision)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.precision (8) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 8:precision: ", p), err) } - } - return err + if p.IsSetPrecision() { + if err := oprot.WriteFieldBegin(ctx, "precision", thrift.I32, 8); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 8:precision: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(*p.Precision)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.precision (8) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 8:precision: ", p), err) + } + } + return err } func (p *SchemaElement) writeField9(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetFieldID() { - if err := oprot.WriteFieldBegin(ctx, "field_id", thrift.I32, 9); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 9:field_id: ", p), err) } - if err := oprot.WriteI32(ctx, int32(*p.FieldID)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.field_id (9) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 9:field_id: ", p), err) } - } - return err + if p.IsSetFieldID() { + if err := oprot.WriteFieldBegin(ctx, "field_id", thrift.I32, 9); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 9:field_id: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(*p.FieldID)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.field_id (9) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 9:field_id: ", p), err) + } + } + return err } func (p *SchemaElement) writeField10(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetLogicalType() { - if err := oprot.WriteFieldBegin(ctx, "logicalType", thrift.STRUCT, 10); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 10:logicalType: ", p), err) } - if err := p.LogicalType.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.LogicalType), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 10:logicalType: ", p), err) } - } - return err + if p.IsSetLogicalType() { + if err := oprot.WriteFieldBegin(ctx, "logicalType", thrift.STRUCT, 10); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 10:logicalType: ", p), err) + } + if err := p.LogicalType.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.LogicalType), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 10:logicalType: ", p), err) + } + } + return err } func (p *SchemaElement) Equals(other *SchemaElement) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if p.Type != other.Type { - if p.Type == nil || other.Type == nil { - return false - } - if (*p.Type) != (*other.Type) { return false } - } - if p.TypeLength != other.TypeLength { - if p.TypeLength == nil || other.TypeLength == nil { - return false - } - if (*p.TypeLength) != (*other.TypeLength) { return false } - } - if p.RepetitionType != other.RepetitionType { - if p.RepetitionType == nil || other.RepetitionType == nil { - return false - } - if (*p.RepetitionType) != (*other.RepetitionType) { return false } - } - if p.Name != other.Name { return false } - if p.NumChildren != other.NumChildren { - if p.NumChildren == nil || other.NumChildren == nil { - return false - } - if (*p.NumChildren) != (*other.NumChildren) { return false } - } - if p.ConvertedType != other.ConvertedType { - if p.ConvertedType == nil || other.ConvertedType == nil { - return false - } - if (*p.ConvertedType) != (*other.ConvertedType) { return false } - } - if p.Scale != other.Scale { - if p.Scale == nil || other.Scale == nil { - return false - } - if (*p.Scale) != (*other.Scale) { return false } - } - if p.Precision != other.Precision { - if p.Precision == nil || other.Precision == nil { - return false - } - if (*p.Precision) != (*other.Precision) { return false } - } - if p.FieldID != other.FieldID { - if p.FieldID == nil || other.FieldID == nil { - return false - } - if (*p.FieldID) != (*other.FieldID) { return false } - } - if !p.LogicalType.Equals(other.LogicalType) { return false } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if p.Type != other.Type { + if p.Type == nil || other.Type == nil { + return false + } + if (*p.Type) != (*other.Type) { + return false + } + } + if p.TypeLength != other.TypeLength { + if p.TypeLength == nil || other.TypeLength == nil { + return false + } + if (*p.TypeLength) != (*other.TypeLength) { + return false + } + } + if p.RepetitionType != other.RepetitionType { + if p.RepetitionType == nil || other.RepetitionType == nil { + return false + } + if (*p.RepetitionType) != (*other.RepetitionType) { + return false + } + } + if p.Name != other.Name { + return false + } + if p.NumChildren != other.NumChildren { + if p.NumChildren == nil || other.NumChildren == nil { + return false + } + if (*p.NumChildren) != (*other.NumChildren) { + return false + } + } + if p.ConvertedType != other.ConvertedType { + if p.ConvertedType == nil || other.ConvertedType == nil { + return false + } + if (*p.ConvertedType) != (*other.ConvertedType) { + return false + } + } + if p.Scale != other.Scale { + if p.Scale == nil || other.Scale == nil { + return false + } + if (*p.Scale) != (*other.Scale) { + return false + } + } + if p.Precision != other.Precision { + if p.Precision == nil || other.Precision == nil { + return false + } + if (*p.Precision) != (*other.Precision) { + return false + } + } + if p.FieldID != other.FieldID { + if p.FieldID == nil || other.FieldID == nil { + return false + } + if (*p.FieldID) != (*other.FieldID) { + return false + } + } + if !p.LogicalType.Equals(other.LogicalType) { + return false + } + return true } func (p *SchemaElement) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("SchemaElement(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("SchemaElement(%+v)", *p) } func (p *SchemaElement) Validate() error { - return nil + return nil } + // Data page header -// +// // Attributes: -// - NumValues: Number of values, including NULLs, in this data page. * -// - Encoding: Encoding used for this data page * -// - DefinitionLevelEncoding: Encoding used for definition levels * -// - RepetitionLevelEncoding: Encoding used for repetition levels * -// - Statistics: Optional statistics for the data in this page* +// - NumValues: Number of values, including NULLs, in this data page. * +// - Encoding: Encoding used for this data page * +// - DefinitionLevelEncoding: Encoding used for definition levels * +// - RepetitionLevelEncoding: Encoding used for repetition levels * +// - Statistics: Optional statistics for the data in this page* type DataPageHeader struct { - NumValues int32 `thrift:"num_values,1,required" db:"num_values" json:"num_values"` - Encoding Encoding `thrift:"encoding,2,required" db:"encoding" json:"encoding"` - DefinitionLevelEncoding Encoding `thrift:"definition_level_encoding,3,required" db:"definition_level_encoding" json:"definition_level_encoding"` - RepetitionLevelEncoding Encoding `thrift:"repetition_level_encoding,4,required" db:"repetition_level_encoding" json:"repetition_level_encoding"` - Statistics *Statistics `thrift:"statistics,5" db:"statistics" json:"statistics,omitempty"` + NumValues int32 `thrift:"num_values,1,required" db:"num_values" json:"num_values"` + Encoding Encoding `thrift:"encoding,2,required" db:"encoding" json:"encoding"` + DefinitionLevelEncoding Encoding `thrift:"definition_level_encoding,3,required" db:"definition_level_encoding" json:"definition_level_encoding"` + RepetitionLevelEncoding Encoding `thrift:"repetition_level_encoding,4,required" db:"repetition_level_encoding" json:"repetition_level_encoding"` + Statistics *Statistics `thrift:"statistics,5" db:"statistics" json:"statistics,omitempty"` } func NewDataPageHeader() *DataPageHeader { - return &DataPageHeader{} + return &DataPageHeader{} } - func (p *DataPageHeader) GetNumValues() int32 { - return p.NumValues + return p.NumValues } func (p *DataPageHeader) GetEncoding() Encoding { - return p.Encoding + return p.Encoding } func (p *DataPageHeader) GetDefinitionLevelEncoding() Encoding { - return p.DefinitionLevelEncoding + return p.DefinitionLevelEncoding } func (p *DataPageHeader) GetRepetitionLevelEncoding() Encoding { - return p.RepetitionLevelEncoding + return p.RepetitionLevelEncoding } + var DataPageHeader_Statistics_DEFAULT *Statistics + func (p *DataPageHeader) GetStatistics() *Statistics { - if !p.IsSetStatistics() { - return DataPageHeader_Statistics_DEFAULT - } -return p.Statistics + if !p.IsSetStatistics() { + return DataPageHeader_Statistics_DEFAULT + } + return p.Statistics } func (p *DataPageHeader) IsSetStatistics() bool { - return p.Statistics != nil + return p.Statistics != nil } func (p *DataPageHeader) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - var issetNumValues bool = false; - var issetEncoding bool = false; - var issetDefinitionLevelEncoding bool = false; - var issetRepetitionLevelEncoding bool = false; - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.I32 { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - issetNumValues = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.I32 { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - issetEncoding = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 3: - if fieldTypeId == thrift.I32 { - if err := p.ReadField3(ctx, iprot); err != nil { - return err - } - issetDefinitionLevelEncoding = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 4: - if fieldTypeId == thrift.I32 { - if err := p.ReadField4(ctx, iprot); err != nil { - return err - } - issetRepetitionLevelEncoding = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 5: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField5(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - if !issetNumValues{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field NumValues is not set")); - } - if !issetEncoding{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Encoding is not set")); - } - if !issetDefinitionLevelEncoding{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field DefinitionLevelEncoding is not set")); - } - if !issetRepetitionLevelEncoding{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field RepetitionLevelEncoding is not set")); - } - return nil -} - -func (p *DataPageHeader) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 1: ", err) -} else { - p.NumValues = v -} - return nil -} - -func (p *DataPageHeader) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 2: ", err) -} else { - temp := Encoding(v) - p.Encoding = temp -} - return nil -} - -func (p *DataPageHeader) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 3: ", err) -} else { - temp := Encoding(v) - p.DefinitionLevelEncoding = temp -} - return nil -} - -func (p *DataPageHeader) ReadField4(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 4: ", err) -} else { - temp := Encoding(v) - p.RepetitionLevelEncoding = temp -} - return nil -} - -func (p *DataPageHeader) ReadField5(ctx context.Context, iprot thrift.TProtocol) error { - p.Statistics = &Statistics{} - if err := p.Statistics.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.Statistics), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + var issetNumValues bool = false + var issetEncoding bool = false + var issetDefinitionLevelEncoding bool = false + var issetRepetitionLevelEncoding bool = false + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.I32 { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + issetNumValues = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.I32 { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + issetEncoding = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 3: + if fieldTypeId == thrift.I32 { + if err := p.ReadField3(ctx, iprot); err != nil { + return err + } + issetDefinitionLevelEncoding = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 4: + if fieldTypeId == thrift.I32 { + if err := p.ReadField4(ctx, iprot); err != nil { + return err + } + issetRepetitionLevelEncoding = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 5: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField5(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + if !issetNumValues { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field NumValues is not set")) + } + if !issetEncoding { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Encoding is not set")) + } + if !issetDefinitionLevelEncoding { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field DefinitionLevelEncoding is not set")) + } + if !issetRepetitionLevelEncoding { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field RepetitionLevelEncoding is not set")) + } + return nil +} + +func (p *DataPageHeader) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 1: ", err) + } else { + p.NumValues = v + } + return nil +} + +func (p *DataPageHeader) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 2: ", err) + } else { + temp := Encoding(v) + p.Encoding = temp + } + return nil +} + +func (p *DataPageHeader) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 3: ", err) + } else { + temp := Encoding(v) + p.DefinitionLevelEncoding = temp + } + return nil +} + +func (p *DataPageHeader) ReadField4(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 4: ", err) + } else { + temp := Encoding(v) + p.RepetitionLevelEncoding = temp + } + return nil +} + +func (p *DataPageHeader) ReadField5(ctx context.Context, iprot thrift.TProtocol) error { + p.Statistics = &Statistics{} + if err := p.Statistics.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.Statistics), err) + } + return nil } func (p *DataPageHeader) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "DataPageHeader"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - if err := p.writeField3(ctx, oprot); err != nil { return err } - if err := p.writeField4(ctx, oprot); err != nil { return err } - if err := p.writeField5(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "DataPageHeader"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + if err := p.writeField3(ctx, oprot); err != nil { + return err + } + if err := p.writeField4(ctx, oprot); err != nil { + return err + } + if err := p.writeField5(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *DataPageHeader) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "num_values", thrift.I32, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:num_values: ", p), err) } - if err := oprot.WriteI32(ctx, int32(p.NumValues)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.num_values (1) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:num_values: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "num_values", thrift.I32, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:num_values: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.NumValues)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.num_values (1) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:num_values: ", p), err) + } + return err } func (p *DataPageHeader) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "encoding", thrift.I32, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:encoding: ", p), err) } - if err := oprot.WriteI32(ctx, int32(p.Encoding)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.encoding (2) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:encoding: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "encoding", thrift.I32, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:encoding: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.Encoding)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.encoding (2) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:encoding: ", p), err) + } + return err } func (p *DataPageHeader) writeField3(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "definition_level_encoding", thrift.I32, 3); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:definition_level_encoding: ", p), err) } - if err := oprot.WriteI32(ctx, int32(p.DefinitionLevelEncoding)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.definition_level_encoding (3) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 3:definition_level_encoding: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "definition_level_encoding", thrift.I32, 3); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:definition_level_encoding: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.DefinitionLevelEncoding)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.definition_level_encoding (3) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 3:definition_level_encoding: ", p), err) + } + return err } func (p *DataPageHeader) writeField4(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "repetition_level_encoding", thrift.I32, 4); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:repetition_level_encoding: ", p), err) } - if err := oprot.WriteI32(ctx, int32(p.RepetitionLevelEncoding)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.repetition_level_encoding (4) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 4:repetition_level_encoding: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "repetition_level_encoding", thrift.I32, 4); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:repetition_level_encoding: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.RepetitionLevelEncoding)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.repetition_level_encoding (4) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 4:repetition_level_encoding: ", p), err) + } + return err } func (p *DataPageHeader) writeField5(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetStatistics() { - if err := oprot.WriteFieldBegin(ctx, "statistics", thrift.STRUCT, 5); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:statistics: ", p), err) } - if err := p.Statistics.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.Statistics), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 5:statistics: ", p), err) } - } - return err + if p.IsSetStatistics() { + if err := oprot.WriteFieldBegin(ctx, "statistics", thrift.STRUCT, 5); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:statistics: ", p), err) + } + if err := p.Statistics.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.Statistics), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 5:statistics: ", p), err) + } + } + return err } func (p *DataPageHeader) Equals(other *DataPageHeader) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if p.NumValues != other.NumValues { return false } - if p.Encoding != other.Encoding { return false } - if p.DefinitionLevelEncoding != other.DefinitionLevelEncoding { return false } - if p.RepetitionLevelEncoding != other.RepetitionLevelEncoding { return false } - if !p.Statistics.Equals(other.Statistics) { return false } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if p.NumValues != other.NumValues { + return false + } + if p.Encoding != other.Encoding { + return false + } + if p.DefinitionLevelEncoding != other.DefinitionLevelEncoding { + return false + } + if p.RepetitionLevelEncoding != other.RepetitionLevelEncoding { + return false + } + if !p.Statistics.Equals(other.Statistics) { + return false + } + return true } func (p *DataPageHeader) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("DataPageHeader(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("DataPageHeader(%+v)", *p) } func (p *DataPageHeader) Validate() error { - return nil + return nil } + type IndexPageHeader struct { } func NewIndexPageHeader() *IndexPageHeader { - return &IndexPageHeader{} + return &IndexPageHeader{} } func (p *IndexPageHeader) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil } func (p *IndexPageHeader) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "IndexPageHeader"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "IndexPageHeader"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *IndexPageHeader) Equals(other *IndexPageHeader) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + return true } func (p *IndexPageHeader) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("IndexPageHeader(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("IndexPageHeader(%+v)", *p) } func (p *IndexPageHeader) Validate() error { - return nil + return nil } + // The dictionary page must be placed at the first position of the column chunk // if it is partly or completely dictionary encoded. At most one dictionary page // can be placed in a column chunk. -// -// +// // Attributes: -// - NumValues: Number of values in the dictionary * -// - Encoding: Encoding using this dictionary page * -// - IsSorted: If true, the entries in the dictionary are sorted in ascending order * +// - NumValues: Number of values in the dictionary * +// - Encoding: Encoding using this dictionary page * +// - IsSorted: If true, the entries in the dictionary are sorted in ascending order * type DictionaryPageHeader struct { - NumValues int32 `thrift:"num_values,1,required" db:"num_values" json:"num_values"` - Encoding Encoding `thrift:"encoding,2,required" db:"encoding" json:"encoding"` - IsSorted *bool `thrift:"is_sorted,3" db:"is_sorted" json:"is_sorted,omitempty"` + NumValues int32 `thrift:"num_values,1,required" db:"num_values" json:"num_values"` + Encoding Encoding `thrift:"encoding,2,required" db:"encoding" json:"encoding"` + IsSorted *bool `thrift:"is_sorted,3" db:"is_sorted" json:"is_sorted,omitempty"` } func NewDictionaryPageHeader() *DictionaryPageHeader { - return &DictionaryPageHeader{} + return &DictionaryPageHeader{} } - func (p *DictionaryPageHeader) GetNumValues() int32 { - return p.NumValues + return p.NumValues } func (p *DictionaryPageHeader) GetEncoding() Encoding { - return p.Encoding + return p.Encoding } + var DictionaryPageHeader_IsSorted_DEFAULT bool + func (p *DictionaryPageHeader) GetIsSorted() bool { - if !p.IsSetIsSorted() { - return DictionaryPageHeader_IsSorted_DEFAULT - } -return *p.IsSorted + if !p.IsSetIsSorted() { + return DictionaryPageHeader_IsSorted_DEFAULT + } + return *p.IsSorted } func (p *DictionaryPageHeader) IsSetIsSorted() bool { - return p.IsSorted != nil + return p.IsSorted != nil } func (p *DictionaryPageHeader) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - var issetNumValues bool = false; - var issetEncoding bool = false; - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.I32 { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - issetNumValues = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.I32 { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - issetEncoding = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 3: - if fieldTypeId == thrift.BOOL { - if err := p.ReadField3(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - if !issetNumValues{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field NumValues is not set")); - } - if !issetEncoding{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Encoding is not set")); - } - return nil -} - -func (p *DictionaryPageHeader) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 1: ", err) -} else { - p.NumValues = v -} - return nil -} - -func (p *DictionaryPageHeader) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 2: ", err) -} else { - temp := Encoding(v) - p.Encoding = temp -} - return nil -} - -func (p *DictionaryPageHeader) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadBool(ctx); err != nil { - return thrift.PrependError("error reading field 3: ", err) -} else { - p.IsSorted = &v -} - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + var issetNumValues bool = false + var issetEncoding bool = false + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.I32 { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + issetNumValues = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.I32 { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + issetEncoding = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 3: + if fieldTypeId == thrift.BOOL { + if err := p.ReadField3(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + if !issetNumValues { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field NumValues is not set")) + } + if !issetEncoding { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Encoding is not set")) + } + return nil +} + +func (p *DictionaryPageHeader) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 1: ", err) + } else { + p.NumValues = v + } + return nil +} + +func (p *DictionaryPageHeader) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 2: ", err) + } else { + temp := Encoding(v) + p.Encoding = temp + } + return nil +} + +func (p *DictionaryPageHeader) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadBool(ctx); err != nil { + return thrift.PrependError("error reading field 3: ", err) + } else { + p.IsSorted = &v + } + return nil } func (p *DictionaryPageHeader) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "DictionaryPageHeader"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - if err := p.writeField3(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "DictionaryPageHeader"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + if err := p.writeField3(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *DictionaryPageHeader) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "num_values", thrift.I32, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:num_values: ", p), err) } - if err := oprot.WriteI32(ctx, int32(p.NumValues)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.num_values (1) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:num_values: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "num_values", thrift.I32, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:num_values: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.NumValues)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.num_values (1) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:num_values: ", p), err) + } + return err } func (p *DictionaryPageHeader) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "encoding", thrift.I32, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:encoding: ", p), err) } - if err := oprot.WriteI32(ctx, int32(p.Encoding)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.encoding (2) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:encoding: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "encoding", thrift.I32, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:encoding: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.Encoding)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.encoding (2) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:encoding: ", p), err) + } + return err } func (p *DictionaryPageHeader) writeField3(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetIsSorted() { - if err := oprot.WriteFieldBegin(ctx, "is_sorted", thrift.BOOL, 3); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:is_sorted: ", p), err) } - if err := oprot.WriteBool(ctx, bool(*p.IsSorted)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.is_sorted (3) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 3:is_sorted: ", p), err) } - } - return err + if p.IsSetIsSorted() { + if err := oprot.WriteFieldBegin(ctx, "is_sorted", thrift.BOOL, 3); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:is_sorted: ", p), err) + } + if err := oprot.WriteBool(ctx, bool(*p.IsSorted)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.is_sorted (3) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 3:is_sorted: ", p), err) + } + } + return err } func (p *DictionaryPageHeader) Equals(other *DictionaryPageHeader) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if p.NumValues != other.NumValues { return false } - if p.Encoding != other.Encoding { return false } - if p.IsSorted != other.IsSorted { - if p.IsSorted == nil || other.IsSorted == nil { - return false - } - if (*p.IsSorted) != (*other.IsSorted) { return false } - } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if p.NumValues != other.NumValues { + return false + } + if p.Encoding != other.Encoding { + return false + } + if p.IsSorted != other.IsSorted { + if p.IsSorted == nil || other.IsSorted == nil { + return false + } + if (*p.IsSorted) != (*other.IsSorted) { + return false + } + } + return true } func (p *DictionaryPageHeader) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("DictionaryPageHeader(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("DictionaryPageHeader(%+v)", *p) } func (p *DictionaryPageHeader) Validate() error { - return nil + return nil } + // New page format allowing reading levels without decompressing the data // Repetition and definition levels are uncompressed // The remaining section containing the data is compressed if is_compressed is true -// -// +// // Attributes: -// - NumValues: Number of values, including NULLs, in this data page. * -// - NumNulls: Number of NULL values, in this data page. +// - NumValues: Number of values, including NULLs, in this data page. * +// - NumNulls: Number of NULL values, in this data page. +// // Number of non-null = num_values - num_nulls which is also the number of values in the data section * -// - NumRows: Number of rows in this data page. which means pages change on record boundaries (r = 0) * -// - Encoding: Encoding used for data in this page * -// - DefinitionLevelsByteLength: length of the definition levels -// - RepetitionLevelsByteLength: length of the repetition levels -// - IsCompressed: whether the values are compressed. +// - NumRows: Number of rows in this data page. which means pages change on record boundaries (r = 0) * +// - Encoding: Encoding used for data in this page * +// - DefinitionLevelsByteLength: length of the definition levels +// - RepetitionLevelsByteLength: length of the repetition levels +// - IsCompressed: whether the values are compressed. +// // Which means the section of the page between // definition_levels_byte_length + repetition_levels_byte_length + 1 and compressed_page_size (included) // is compressed with the compression_codec. // If missing it is considered compressed -// - Statistics: optional statistics for the data in this page * +// - Statistics: optional statistics for the data in this page * type DataPageHeaderV2 struct { - NumValues int32 `thrift:"num_values,1,required" db:"num_values" json:"num_values"` - NumNulls int32 `thrift:"num_nulls,2,required" db:"num_nulls" json:"num_nulls"` - NumRows int32 `thrift:"num_rows,3,required" db:"num_rows" json:"num_rows"` - Encoding Encoding `thrift:"encoding,4,required" db:"encoding" json:"encoding"` - DefinitionLevelsByteLength int32 `thrift:"definition_levels_byte_length,5,required" db:"definition_levels_byte_length" json:"definition_levels_byte_length"` - RepetitionLevelsByteLength int32 `thrift:"repetition_levels_byte_length,6,required" db:"repetition_levels_byte_length" json:"repetition_levels_byte_length"` - IsCompressed bool `thrift:"is_compressed,7" db:"is_compressed" json:"is_compressed"` - Statistics *Statistics `thrift:"statistics,8" db:"statistics" json:"statistics,omitempty"` + NumValues int32 `thrift:"num_values,1,required" db:"num_values" json:"num_values"` + NumNulls int32 `thrift:"num_nulls,2,required" db:"num_nulls" json:"num_nulls"` + NumRows int32 `thrift:"num_rows,3,required" db:"num_rows" json:"num_rows"` + Encoding Encoding `thrift:"encoding,4,required" db:"encoding" json:"encoding"` + DefinitionLevelsByteLength int32 `thrift:"definition_levels_byte_length,5,required" db:"definition_levels_byte_length" json:"definition_levels_byte_length"` + RepetitionLevelsByteLength int32 `thrift:"repetition_levels_byte_length,6,required" db:"repetition_levels_byte_length" json:"repetition_levels_byte_length"` + IsCompressed bool `thrift:"is_compressed,7" db:"is_compressed" json:"is_compressed"` + Statistics *Statistics `thrift:"statistics,8" db:"statistics" json:"statistics,omitempty"` } func NewDataPageHeaderV2() *DataPageHeaderV2 { - return &DataPageHeaderV2{ -IsCompressed: true, -} + return &DataPageHeaderV2{ + IsCompressed: true, + } } - func (p *DataPageHeaderV2) GetNumValues() int32 { - return p.NumValues + return p.NumValues } func (p *DataPageHeaderV2) GetNumNulls() int32 { - return p.NumNulls + return p.NumNulls } func (p *DataPageHeaderV2) GetNumRows() int32 { - return p.NumRows + return p.NumRows } func (p *DataPageHeaderV2) GetEncoding() Encoding { - return p.Encoding + return p.Encoding } func (p *DataPageHeaderV2) GetDefinitionLevelsByteLength() int32 { - return p.DefinitionLevelsByteLength + return p.DefinitionLevelsByteLength } func (p *DataPageHeaderV2) GetRepetitionLevelsByteLength() int32 { - return p.RepetitionLevelsByteLength + return p.RepetitionLevelsByteLength } + var DataPageHeaderV2_IsCompressed_DEFAULT bool = true func (p *DataPageHeaderV2) GetIsCompressed() bool { - return p.IsCompressed + return p.IsCompressed } + var DataPageHeaderV2_Statistics_DEFAULT *Statistics + func (p *DataPageHeaderV2) GetStatistics() *Statistics { - if !p.IsSetStatistics() { - return DataPageHeaderV2_Statistics_DEFAULT - } -return p.Statistics + if !p.IsSetStatistics() { + return DataPageHeaderV2_Statistics_DEFAULT + } + return p.Statistics } func (p *DataPageHeaderV2) IsSetIsCompressed() bool { - return p.IsCompressed != DataPageHeaderV2_IsCompressed_DEFAULT + return p.IsCompressed != DataPageHeaderV2_IsCompressed_DEFAULT } func (p *DataPageHeaderV2) IsSetStatistics() bool { - return p.Statistics != nil + return p.Statistics != nil } func (p *DataPageHeaderV2) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - var issetNumValues bool = false; - var issetNumNulls bool = false; - var issetNumRows bool = false; - var issetEncoding bool = false; - var issetDefinitionLevelsByteLength bool = false; - var issetRepetitionLevelsByteLength bool = false; - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.I32 { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - issetNumValues = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.I32 { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - issetNumNulls = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 3: - if fieldTypeId == thrift.I32 { - if err := p.ReadField3(ctx, iprot); err != nil { - return err - } - issetNumRows = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 4: - if fieldTypeId == thrift.I32 { - if err := p.ReadField4(ctx, iprot); err != nil { - return err - } - issetEncoding = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 5: - if fieldTypeId == thrift.I32 { - if err := p.ReadField5(ctx, iprot); err != nil { - return err - } - issetDefinitionLevelsByteLength = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 6: - if fieldTypeId == thrift.I32 { - if err := p.ReadField6(ctx, iprot); err != nil { - return err - } - issetRepetitionLevelsByteLength = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 7: - if fieldTypeId == thrift.BOOL { - if err := p.ReadField7(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 8: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField8(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - if !issetNumValues{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field NumValues is not set")); - } - if !issetNumNulls{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field NumNulls is not set")); - } - if !issetNumRows{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field NumRows is not set")); - } - if !issetEncoding{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Encoding is not set")); - } - if !issetDefinitionLevelsByteLength{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field DefinitionLevelsByteLength is not set")); - } - if !issetRepetitionLevelsByteLength{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field RepetitionLevelsByteLength is not set")); - } - return nil -} - -func (p *DataPageHeaderV2) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 1: ", err) -} else { - p.NumValues = v -} - return nil -} - -func (p *DataPageHeaderV2) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 2: ", err) -} else { - p.NumNulls = v -} - return nil -} - -func (p *DataPageHeaderV2) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 3: ", err) -} else { - p.NumRows = v -} - return nil -} - -func (p *DataPageHeaderV2) ReadField4(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 4: ", err) -} else { - temp := Encoding(v) - p.Encoding = temp -} - return nil -} - -func (p *DataPageHeaderV2) ReadField5(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 5: ", err) -} else { - p.DefinitionLevelsByteLength = v -} - return nil -} - -func (p *DataPageHeaderV2) ReadField6(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 6: ", err) -} else { - p.RepetitionLevelsByteLength = v -} - return nil -} - -func (p *DataPageHeaderV2) ReadField7(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadBool(ctx); err != nil { - return thrift.PrependError("error reading field 7: ", err) -} else { - p.IsCompressed = v -} - return nil -} - -func (p *DataPageHeaderV2) ReadField8(ctx context.Context, iprot thrift.TProtocol) error { - p.Statistics = &Statistics{} - if err := p.Statistics.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.Statistics), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + var issetNumValues bool = false + var issetNumNulls bool = false + var issetNumRows bool = false + var issetEncoding bool = false + var issetDefinitionLevelsByteLength bool = false + var issetRepetitionLevelsByteLength bool = false + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.I32 { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + issetNumValues = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.I32 { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + issetNumNulls = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 3: + if fieldTypeId == thrift.I32 { + if err := p.ReadField3(ctx, iprot); err != nil { + return err + } + issetNumRows = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 4: + if fieldTypeId == thrift.I32 { + if err := p.ReadField4(ctx, iprot); err != nil { + return err + } + issetEncoding = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 5: + if fieldTypeId == thrift.I32 { + if err := p.ReadField5(ctx, iprot); err != nil { + return err + } + issetDefinitionLevelsByteLength = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 6: + if fieldTypeId == thrift.I32 { + if err := p.ReadField6(ctx, iprot); err != nil { + return err + } + issetRepetitionLevelsByteLength = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 7: + if fieldTypeId == thrift.BOOL { + if err := p.ReadField7(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 8: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField8(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + if !issetNumValues { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field NumValues is not set")) + } + if !issetNumNulls { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field NumNulls is not set")) + } + if !issetNumRows { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field NumRows is not set")) + } + if !issetEncoding { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Encoding is not set")) + } + if !issetDefinitionLevelsByteLength { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field DefinitionLevelsByteLength is not set")) + } + if !issetRepetitionLevelsByteLength { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field RepetitionLevelsByteLength is not set")) + } + return nil +} + +func (p *DataPageHeaderV2) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 1: ", err) + } else { + p.NumValues = v + } + return nil +} + +func (p *DataPageHeaderV2) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 2: ", err) + } else { + p.NumNulls = v + } + return nil +} + +func (p *DataPageHeaderV2) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 3: ", err) + } else { + p.NumRows = v + } + return nil +} + +func (p *DataPageHeaderV2) ReadField4(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 4: ", err) + } else { + temp := Encoding(v) + p.Encoding = temp + } + return nil +} + +func (p *DataPageHeaderV2) ReadField5(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 5: ", err) + } else { + p.DefinitionLevelsByteLength = v + } + return nil +} + +func (p *DataPageHeaderV2) ReadField6(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 6: ", err) + } else { + p.RepetitionLevelsByteLength = v + } + return nil +} + +func (p *DataPageHeaderV2) ReadField7(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadBool(ctx); err != nil { + return thrift.PrependError("error reading field 7: ", err) + } else { + p.IsCompressed = v + } + return nil +} + +func (p *DataPageHeaderV2) ReadField8(ctx context.Context, iprot thrift.TProtocol) error { + p.Statistics = &Statistics{} + if err := p.Statistics.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.Statistics), err) + } + return nil } func (p *DataPageHeaderV2) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "DataPageHeaderV2"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - if err := p.writeField3(ctx, oprot); err != nil { return err } - if err := p.writeField4(ctx, oprot); err != nil { return err } - if err := p.writeField5(ctx, oprot); err != nil { return err } - if err := p.writeField6(ctx, oprot); err != nil { return err } - if err := p.writeField7(ctx, oprot); err != nil { return err } - if err := p.writeField8(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "DataPageHeaderV2"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + if err := p.writeField3(ctx, oprot); err != nil { + return err + } + if err := p.writeField4(ctx, oprot); err != nil { + return err + } + if err := p.writeField5(ctx, oprot); err != nil { + return err + } + if err := p.writeField6(ctx, oprot); err != nil { + return err + } + if err := p.writeField7(ctx, oprot); err != nil { + return err + } + if err := p.writeField8(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *DataPageHeaderV2) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "num_values", thrift.I32, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:num_values: ", p), err) } - if err := oprot.WriteI32(ctx, int32(p.NumValues)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.num_values (1) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:num_values: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "num_values", thrift.I32, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:num_values: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.NumValues)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.num_values (1) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:num_values: ", p), err) + } + return err } func (p *DataPageHeaderV2) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "num_nulls", thrift.I32, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:num_nulls: ", p), err) } - if err := oprot.WriteI32(ctx, int32(p.NumNulls)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.num_nulls (2) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:num_nulls: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "num_nulls", thrift.I32, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:num_nulls: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.NumNulls)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.num_nulls (2) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:num_nulls: ", p), err) + } + return err } func (p *DataPageHeaderV2) writeField3(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "num_rows", thrift.I32, 3); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:num_rows: ", p), err) } - if err := oprot.WriteI32(ctx, int32(p.NumRows)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.num_rows (3) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 3:num_rows: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "num_rows", thrift.I32, 3); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:num_rows: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.NumRows)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.num_rows (3) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 3:num_rows: ", p), err) + } + return err } func (p *DataPageHeaderV2) writeField4(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "encoding", thrift.I32, 4); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:encoding: ", p), err) } - if err := oprot.WriteI32(ctx, int32(p.Encoding)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.encoding (4) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 4:encoding: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "encoding", thrift.I32, 4); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:encoding: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.Encoding)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.encoding (4) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 4:encoding: ", p), err) + } + return err } func (p *DataPageHeaderV2) writeField5(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "definition_levels_byte_length", thrift.I32, 5); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:definition_levels_byte_length: ", p), err) } - if err := oprot.WriteI32(ctx, int32(p.DefinitionLevelsByteLength)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.definition_levels_byte_length (5) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 5:definition_levels_byte_length: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "definition_levels_byte_length", thrift.I32, 5); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:definition_levels_byte_length: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.DefinitionLevelsByteLength)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.definition_levels_byte_length (5) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 5:definition_levels_byte_length: ", p), err) + } + return err } func (p *DataPageHeaderV2) writeField6(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "repetition_levels_byte_length", thrift.I32, 6); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 6:repetition_levels_byte_length: ", p), err) } - if err := oprot.WriteI32(ctx, int32(p.RepetitionLevelsByteLength)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.repetition_levels_byte_length (6) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 6:repetition_levels_byte_length: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "repetition_levels_byte_length", thrift.I32, 6); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 6:repetition_levels_byte_length: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.RepetitionLevelsByteLength)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.repetition_levels_byte_length (6) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 6:repetition_levels_byte_length: ", p), err) + } + return err } func (p *DataPageHeaderV2) writeField7(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetIsCompressed() { - if err := oprot.WriteFieldBegin(ctx, "is_compressed", thrift.BOOL, 7); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 7:is_compressed: ", p), err) } - if err := oprot.WriteBool(ctx, bool(p.IsCompressed)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.is_compressed (7) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 7:is_compressed: ", p), err) } - } - return err + if p.IsSetIsCompressed() { + if err := oprot.WriteFieldBegin(ctx, "is_compressed", thrift.BOOL, 7); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 7:is_compressed: ", p), err) + } + if err := oprot.WriteBool(ctx, bool(p.IsCompressed)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.is_compressed (7) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 7:is_compressed: ", p), err) + } + } + return err } func (p *DataPageHeaderV2) writeField8(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetStatistics() { - if err := oprot.WriteFieldBegin(ctx, "statistics", thrift.STRUCT, 8); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 8:statistics: ", p), err) } - if err := p.Statistics.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.Statistics), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 8:statistics: ", p), err) } - } - return err + if p.IsSetStatistics() { + if err := oprot.WriteFieldBegin(ctx, "statistics", thrift.STRUCT, 8); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 8:statistics: ", p), err) + } + if err := p.Statistics.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.Statistics), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 8:statistics: ", p), err) + } + } + return err } func (p *DataPageHeaderV2) Equals(other *DataPageHeaderV2) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if p.NumValues != other.NumValues { return false } - if p.NumNulls != other.NumNulls { return false } - if p.NumRows != other.NumRows { return false } - if p.Encoding != other.Encoding { return false } - if p.DefinitionLevelsByteLength != other.DefinitionLevelsByteLength { return false } - if p.RepetitionLevelsByteLength != other.RepetitionLevelsByteLength { return false } - if p.IsCompressed != other.IsCompressed { return false } - if !p.Statistics.Equals(other.Statistics) { return false } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if p.NumValues != other.NumValues { + return false + } + if p.NumNulls != other.NumNulls { + return false + } + if p.NumRows != other.NumRows { + return false + } + if p.Encoding != other.Encoding { + return false + } + if p.DefinitionLevelsByteLength != other.DefinitionLevelsByteLength { + return false + } + if p.RepetitionLevelsByteLength != other.RepetitionLevelsByteLength { + return false + } + if p.IsCompressed != other.IsCompressed { + return false + } + if !p.Statistics.Equals(other.Statistics) { + return false + } + return true } func (p *DataPageHeaderV2) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("DataPageHeaderV2(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("DataPageHeaderV2(%+v)", *p) } func (p *DataPageHeaderV2) Validate() error { - return nil + return nil } + // Block-based algorithm type annotation. * type SplitBlockAlgorithm struct { } func NewSplitBlockAlgorithm() *SplitBlockAlgorithm { - return &SplitBlockAlgorithm{} + return &SplitBlockAlgorithm{} } func (p *SplitBlockAlgorithm) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil } func (p *SplitBlockAlgorithm) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "SplitBlockAlgorithm"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "SplitBlockAlgorithm"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *SplitBlockAlgorithm) Equals(other *SplitBlockAlgorithm) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + return true } func (p *SplitBlockAlgorithm) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("SplitBlockAlgorithm(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("SplitBlockAlgorithm(%+v)", *p) } func (p *SplitBlockAlgorithm) Validate() error { - return nil + return nil } + // The algorithm used in Bloom filter. * -// +// // Attributes: -// - BLOCK: Block-based Bloom filter. * +// - BLOCK: Block-based Bloom filter. * type BloomFilterAlgorithm struct { - BLOCK *SplitBlockAlgorithm `thrift:"BLOCK,1" db:"BLOCK" json:"BLOCK,omitempty"` + BLOCK *SplitBlockAlgorithm `thrift:"BLOCK,1" db:"BLOCK" json:"BLOCK,omitempty"` } func NewBloomFilterAlgorithm() *BloomFilterAlgorithm { - return &BloomFilterAlgorithm{} + return &BloomFilterAlgorithm{} } var BloomFilterAlgorithm_BLOCK_DEFAULT *SplitBlockAlgorithm + func (p *BloomFilterAlgorithm) GetBLOCK() *SplitBlockAlgorithm { - if !p.IsSetBLOCK() { - return BloomFilterAlgorithm_BLOCK_DEFAULT - } -return p.BLOCK + if !p.IsSetBLOCK() { + return BloomFilterAlgorithm_BLOCK_DEFAULT + } + return p.BLOCK } func (p *BloomFilterAlgorithm) CountSetFieldsBloomFilterAlgorithm() int { - count := 0 - if (p.IsSetBLOCK()) { - count++ - } - return count + count := 0 + if p.IsSetBLOCK() { + count++ + } + return count } func (p *BloomFilterAlgorithm) IsSetBLOCK() bool { - return p.BLOCK != nil + return p.BLOCK != nil } func (p *BloomFilterAlgorithm) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil -} - -func (p *BloomFilterAlgorithm) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - p.BLOCK = &SplitBlockAlgorithm{} - if err := p.BLOCK.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.BLOCK), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil +} + +func (p *BloomFilterAlgorithm) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + p.BLOCK = &SplitBlockAlgorithm{} + if err := p.BLOCK.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.BLOCK), err) + } + return nil } func (p *BloomFilterAlgorithm) Write(ctx context.Context, oprot thrift.TProtocol) error { - if c := p.CountSetFieldsBloomFilterAlgorithm(); c != 1 { - return fmt.Errorf("%T write union: exactly one field must be set (%d set)", p, c) - } - if err := oprot.WriteStructBegin(ctx, "BloomFilterAlgorithm"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if c := p.CountSetFieldsBloomFilterAlgorithm(); c != 1 { + return fmt.Errorf("%T write union: exactly one field must be set (%d set)", p, c) + } + if err := oprot.WriteStructBegin(ctx, "BloomFilterAlgorithm"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *BloomFilterAlgorithm) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetBLOCK() { - if err := oprot.WriteFieldBegin(ctx, "BLOCK", thrift.STRUCT, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:BLOCK: ", p), err) } - if err := p.BLOCK.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.BLOCK), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:BLOCK: ", p), err) } - } - return err + if p.IsSetBLOCK() { + if err := oprot.WriteFieldBegin(ctx, "BLOCK", thrift.STRUCT, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:BLOCK: ", p), err) + } + if err := p.BLOCK.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.BLOCK), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:BLOCK: ", p), err) + } + } + return err } func (p *BloomFilterAlgorithm) Equals(other *BloomFilterAlgorithm) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if !p.BLOCK.Equals(other.BLOCK) { return false } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if !p.BLOCK.Equals(other.BLOCK) { + return false + } + return true } func (p *BloomFilterAlgorithm) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("BloomFilterAlgorithm(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("BloomFilterAlgorithm(%+v)", *p) } func (p *BloomFilterAlgorithm) Validate() error { - return nil + return nil } + // Hash strategy type annotation. xxHash is an extremely fast non-cryptographic hash // algorithm. It uses 64 bits version of xxHash. -// type XxHash struct { } func NewXxHash() *XxHash { - return &XxHash{} + return &XxHash{} } func (p *XxHash) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil } func (p *XxHash) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "XxHash"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "XxHash"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *XxHash) Equals(other *XxHash) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + return true } func (p *XxHash) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("XxHash(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("XxHash(%+v)", *p) } func (p *XxHash) Validate() error { - return nil + return nil } + // The hash function used in Bloom filter. This function takes the hash of a column value // using plain encoding. -// -// +// // Attributes: -// - XXHASH: xxHash Strategy. * +// - XXHASH: xxHash Strategy. * type BloomFilterHash struct { - XXHASH *XxHash `thrift:"XXHASH,1" db:"XXHASH" json:"XXHASH,omitempty"` + XXHASH *XxHash `thrift:"XXHASH,1" db:"XXHASH" json:"XXHASH,omitempty"` } func NewBloomFilterHash() *BloomFilterHash { - return &BloomFilterHash{} + return &BloomFilterHash{} } var BloomFilterHash_XXHASH_DEFAULT *XxHash + func (p *BloomFilterHash) GetXXHASH() *XxHash { - if !p.IsSetXXHASH() { - return BloomFilterHash_XXHASH_DEFAULT - } -return p.XXHASH + if !p.IsSetXXHASH() { + return BloomFilterHash_XXHASH_DEFAULT + } + return p.XXHASH } func (p *BloomFilterHash) CountSetFieldsBloomFilterHash() int { - count := 0 - if (p.IsSetXXHASH()) { - count++ - } - return count + count := 0 + if p.IsSetXXHASH() { + count++ + } + return count } func (p *BloomFilterHash) IsSetXXHASH() bool { - return p.XXHASH != nil + return p.XXHASH != nil } func (p *BloomFilterHash) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil -} - -func (p *BloomFilterHash) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - p.XXHASH = &XxHash{} - if err := p.XXHASH.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.XXHASH), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil +} + +func (p *BloomFilterHash) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + p.XXHASH = &XxHash{} + if err := p.XXHASH.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.XXHASH), err) + } + return nil } func (p *BloomFilterHash) Write(ctx context.Context, oprot thrift.TProtocol) error { - if c := p.CountSetFieldsBloomFilterHash(); c != 1 { - return fmt.Errorf("%T write union: exactly one field must be set (%d set)", p, c) - } - if err := oprot.WriteStructBegin(ctx, "BloomFilterHash"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if c := p.CountSetFieldsBloomFilterHash(); c != 1 { + return fmt.Errorf("%T write union: exactly one field must be set (%d set)", p, c) + } + if err := oprot.WriteStructBegin(ctx, "BloomFilterHash"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *BloomFilterHash) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetXXHASH() { - if err := oprot.WriteFieldBegin(ctx, "XXHASH", thrift.STRUCT, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:XXHASH: ", p), err) } - if err := p.XXHASH.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.XXHASH), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:XXHASH: ", p), err) } - } - return err + if p.IsSetXXHASH() { + if err := oprot.WriteFieldBegin(ctx, "XXHASH", thrift.STRUCT, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:XXHASH: ", p), err) + } + if err := p.XXHASH.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.XXHASH), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:XXHASH: ", p), err) + } + } + return err } func (p *BloomFilterHash) Equals(other *BloomFilterHash) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if !p.XXHASH.Equals(other.XXHASH) { return false } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if !p.XXHASH.Equals(other.XXHASH) { + return false + } + return true } func (p *BloomFilterHash) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("BloomFilterHash(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("BloomFilterHash(%+v)", *p) } func (p *BloomFilterHash) Validate() error { - return nil + return nil } + // The compression used in the Bloom filter. -// type Uncompressed struct { } func NewUncompressed() *Uncompressed { - return &Uncompressed{} + return &Uncompressed{} } func (p *Uncompressed) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil } func (p *Uncompressed) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "Uncompressed"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "Uncompressed"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *Uncompressed) Equals(other *Uncompressed) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + return true } func (p *Uncompressed) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("Uncompressed(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("Uncompressed(%+v)", *p) } func (p *Uncompressed) Validate() error { - return nil + return nil } + // Attributes: -// - UNCOMPRESSED +// - UNCOMPRESSED type BloomFilterCompression struct { - UNCOMPRESSED *Uncompressed `thrift:"UNCOMPRESSED,1" db:"UNCOMPRESSED" json:"UNCOMPRESSED,omitempty"` + UNCOMPRESSED *Uncompressed `thrift:"UNCOMPRESSED,1" db:"UNCOMPRESSED" json:"UNCOMPRESSED,omitempty"` } func NewBloomFilterCompression() *BloomFilterCompression { - return &BloomFilterCompression{} + return &BloomFilterCompression{} } var BloomFilterCompression_UNCOMPRESSED_DEFAULT *Uncompressed + func (p *BloomFilterCompression) GetUNCOMPRESSED() *Uncompressed { - if !p.IsSetUNCOMPRESSED() { - return BloomFilterCompression_UNCOMPRESSED_DEFAULT - } -return p.UNCOMPRESSED + if !p.IsSetUNCOMPRESSED() { + return BloomFilterCompression_UNCOMPRESSED_DEFAULT + } + return p.UNCOMPRESSED } func (p *BloomFilterCompression) CountSetFieldsBloomFilterCompression() int { - count := 0 - if (p.IsSetUNCOMPRESSED()) { - count++ - } - return count + count := 0 + if p.IsSetUNCOMPRESSED() { + count++ + } + return count } func (p *BloomFilterCompression) IsSetUNCOMPRESSED() bool { - return p.UNCOMPRESSED != nil + return p.UNCOMPRESSED != nil } func (p *BloomFilterCompression) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil -} - -func (p *BloomFilterCompression) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - p.UNCOMPRESSED = &Uncompressed{} - if err := p.UNCOMPRESSED.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.UNCOMPRESSED), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil +} + +func (p *BloomFilterCompression) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + p.UNCOMPRESSED = &Uncompressed{} + if err := p.UNCOMPRESSED.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.UNCOMPRESSED), err) + } + return nil } func (p *BloomFilterCompression) Write(ctx context.Context, oprot thrift.TProtocol) error { - if c := p.CountSetFieldsBloomFilterCompression(); c != 1 { - return fmt.Errorf("%T write union: exactly one field must be set (%d set)", p, c) - } - if err := oprot.WriteStructBegin(ctx, "BloomFilterCompression"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if c := p.CountSetFieldsBloomFilterCompression(); c != 1 { + return fmt.Errorf("%T write union: exactly one field must be set (%d set)", p, c) + } + if err := oprot.WriteStructBegin(ctx, "BloomFilterCompression"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *BloomFilterCompression) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetUNCOMPRESSED() { - if err := oprot.WriteFieldBegin(ctx, "UNCOMPRESSED", thrift.STRUCT, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:UNCOMPRESSED: ", p), err) } - if err := p.UNCOMPRESSED.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.UNCOMPRESSED), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:UNCOMPRESSED: ", p), err) } - } - return err + if p.IsSetUNCOMPRESSED() { + if err := oprot.WriteFieldBegin(ctx, "UNCOMPRESSED", thrift.STRUCT, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:UNCOMPRESSED: ", p), err) + } + if err := p.UNCOMPRESSED.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.UNCOMPRESSED), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:UNCOMPRESSED: ", p), err) + } + } + return err } func (p *BloomFilterCompression) Equals(other *BloomFilterCompression) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if !p.UNCOMPRESSED.Equals(other.UNCOMPRESSED) { return false } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if !p.UNCOMPRESSED.Equals(other.UNCOMPRESSED) { + return false + } + return true } func (p *BloomFilterCompression) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("BloomFilterCompression(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("BloomFilterCompression(%+v)", *p) } func (p *BloomFilterCompression) Validate() error { - return nil + return nil } + // Bloom filter header is stored at beginning of Bloom filter data of each column // and followed by its bitset. -// -// +// // Attributes: -// - NumBytes: The size of bitset in bytes * -// - Algorithm: The algorithm for setting bits. * -// - Hash: The hash function used for Bloom filter. * -// - Compression: The compression used in the Bloom filter * +// - NumBytes: The size of bitset in bytes * +// - Algorithm: The algorithm for setting bits. * +// - Hash: The hash function used for Bloom filter. * +// - Compression: The compression used in the Bloom filter * type BloomFilterHeader struct { - NumBytes int32 `thrift:"numBytes,1,required" db:"numBytes" json:"numBytes"` - Algorithm *BloomFilterAlgorithm `thrift:"algorithm,2,required" db:"algorithm" json:"algorithm"` - Hash *BloomFilterHash `thrift:"hash,3,required" db:"hash" json:"hash"` - Compression *BloomFilterCompression `thrift:"compression,4,required" db:"compression" json:"compression"` + NumBytes int32 `thrift:"numBytes,1,required" db:"numBytes" json:"numBytes"` + Algorithm *BloomFilterAlgorithm `thrift:"algorithm,2,required" db:"algorithm" json:"algorithm"` + Hash *BloomFilterHash `thrift:"hash,3,required" db:"hash" json:"hash"` + Compression *BloomFilterCompression `thrift:"compression,4,required" db:"compression" json:"compression"` } func NewBloomFilterHeader() *BloomFilterHeader { - return &BloomFilterHeader{} + return &BloomFilterHeader{} } - func (p *BloomFilterHeader) GetNumBytes() int32 { - return p.NumBytes + return p.NumBytes } + var BloomFilterHeader_Algorithm_DEFAULT *BloomFilterAlgorithm + func (p *BloomFilterHeader) GetAlgorithm() *BloomFilterAlgorithm { - if !p.IsSetAlgorithm() { - return BloomFilterHeader_Algorithm_DEFAULT - } -return p.Algorithm + if !p.IsSetAlgorithm() { + return BloomFilterHeader_Algorithm_DEFAULT + } + return p.Algorithm } + var BloomFilterHeader_Hash_DEFAULT *BloomFilterHash + func (p *BloomFilterHeader) GetHash() *BloomFilterHash { - if !p.IsSetHash() { - return BloomFilterHeader_Hash_DEFAULT - } -return p.Hash + if !p.IsSetHash() { + return BloomFilterHeader_Hash_DEFAULT + } + return p.Hash } + var BloomFilterHeader_Compression_DEFAULT *BloomFilterCompression + func (p *BloomFilterHeader) GetCompression() *BloomFilterCompression { - if !p.IsSetCompression() { - return BloomFilterHeader_Compression_DEFAULT - } -return p.Compression + if !p.IsSetCompression() { + return BloomFilterHeader_Compression_DEFAULT + } + return p.Compression } func (p *BloomFilterHeader) IsSetAlgorithm() bool { - return p.Algorithm != nil + return p.Algorithm != nil } func (p *BloomFilterHeader) IsSetHash() bool { - return p.Hash != nil + return p.Hash != nil } func (p *BloomFilterHeader) IsSetCompression() bool { - return p.Compression != nil + return p.Compression != nil } func (p *BloomFilterHeader) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - var issetNumBytes bool = false; - var issetAlgorithm bool = false; - var issetHash bool = false; - var issetCompression bool = false; - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.I32 { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - issetNumBytes = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - issetAlgorithm = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 3: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField3(ctx, iprot); err != nil { - return err - } - issetHash = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 4: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField4(ctx, iprot); err != nil { - return err - } - issetCompression = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - if !issetNumBytes{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field NumBytes is not set")); - } - if !issetAlgorithm{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Algorithm is not set")); - } - if !issetHash{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Hash is not set")); - } - if !issetCompression{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Compression is not set")); - } - return nil -} - -func (p *BloomFilterHeader) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 1: ", err) -} else { - p.NumBytes = v -} - return nil -} - -func (p *BloomFilterHeader) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - p.Algorithm = &BloomFilterAlgorithm{} - if err := p.Algorithm.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.Algorithm), err) - } - return nil -} - -func (p *BloomFilterHeader) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { - p.Hash = &BloomFilterHash{} - if err := p.Hash.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.Hash), err) - } - return nil -} - -func (p *BloomFilterHeader) ReadField4(ctx context.Context, iprot thrift.TProtocol) error { - p.Compression = &BloomFilterCompression{} - if err := p.Compression.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.Compression), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + var issetNumBytes bool = false + var issetAlgorithm bool = false + var issetHash bool = false + var issetCompression bool = false + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.I32 { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + issetNumBytes = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + issetAlgorithm = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 3: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField3(ctx, iprot); err != nil { + return err + } + issetHash = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 4: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField4(ctx, iprot); err != nil { + return err + } + issetCompression = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + if !issetNumBytes { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field NumBytes is not set")) + } + if !issetAlgorithm { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Algorithm is not set")) + } + if !issetHash { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Hash is not set")) + } + if !issetCompression { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Compression is not set")) + } + return nil +} + +func (p *BloomFilterHeader) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 1: ", err) + } else { + p.NumBytes = v + } + return nil +} + +func (p *BloomFilterHeader) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + p.Algorithm = &BloomFilterAlgorithm{} + if err := p.Algorithm.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.Algorithm), err) + } + return nil +} + +func (p *BloomFilterHeader) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { + p.Hash = &BloomFilterHash{} + if err := p.Hash.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.Hash), err) + } + return nil +} + +func (p *BloomFilterHeader) ReadField4(ctx context.Context, iprot thrift.TProtocol) error { + p.Compression = &BloomFilterCompression{} + if err := p.Compression.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.Compression), err) + } + return nil } func (p *BloomFilterHeader) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "BloomFilterHeader"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - if err := p.writeField3(ctx, oprot); err != nil { return err } - if err := p.writeField4(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "BloomFilterHeader"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + if err := p.writeField3(ctx, oprot); err != nil { + return err + } + if err := p.writeField4(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *BloomFilterHeader) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "numBytes", thrift.I32, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:numBytes: ", p), err) } - if err := oprot.WriteI32(ctx, int32(p.NumBytes)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.numBytes (1) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:numBytes: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "numBytes", thrift.I32, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:numBytes: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.NumBytes)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.numBytes (1) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:numBytes: ", p), err) + } + return err } func (p *BloomFilterHeader) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "algorithm", thrift.STRUCT, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:algorithm: ", p), err) } - if err := p.Algorithm.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.Algorithm), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:algorithm: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "algorithm", thrift.STRUCT, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:algorithm: ", p), err) + } + if err := p.Algorithm.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.Algorithm), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:algorithm: ", p), err) + } + return err } func (p *BloomFilterHeader) writeField3(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "hash", thrift.STRUCT, 3); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:hash: ", p), err) } - if err := p.Hash.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.Hash), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 3:hash: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "hash", thrift.STRUCT, 3); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:hash: ", p), err) + } + if err := p.Hash.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.Hash), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 3:hash: ", p), err) + } + return err } func (p *BloomFilterHeader) writeField4(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "compression", thrift.STRUCT, 4); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:compression: ", p), err) } - if err := p.Compression.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.Compression), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 4:compression: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "compression", thrift.STRUCT, 4); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:compression: ", p), err) + } + if err := p.Compression.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.Compression), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 4:compression: ", p), err) + } + return err } func (p *BloomFilterHeader) Equals(other *BloomFilterHeader) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if p.NumBytes != other.NumBytes { return false } - if !p.Algorithm.Equals(other.Algorithm) { return false } - if !p.Hash.Equals(other.Hash) { return false } - if !p.Compression.Equals(other.Compression) { return false } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if p.NumBytes != other.NumBytes { + return false + } + if !p.Algorithm.Equals(other.Algorithm) { + return false + } + if !p.Hash.Equals(other.Hash) { + return false + } + if !p.Compression.Equals(other.Compression) { + return false + } + return true } func (p *BloomFilterHeader) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("BloomFilterHeader(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("BloomFilterHeader(%+v)", *p) } func (p *BloomFilterHeader) Validate() error { - return nil + return nil } + // Attributes: -// - Type: the type of the page: indicates which of the *_header fields is set * -// - UncompressedPageSize: Uncompressed page size in bytes (not including this header) * -// - CompressedPageSize: Compressed (and potentially encrypted) page size in bytes, not including this header * -// - Crc: The 32-bit CRC checksum for the page, to be be calculated as follows: -// -// - The standard CRC32 algorithm is used (with polynomial 0x04C11DB7, -// the same as in e.g. GZip). -// - All page types can have a CRC (v1 and v2 data pages, dictionary pages, -// etc.). -// - The CRC is computed on the serialization binary representation of the page -// (as written to disk), excluding the page header. For example, for v1 -// data pages, the CRC is computed on the concatenation of repetition levels, -// definition levels and column values (optionally compressed, optionally -// encrypted). -// - The CRC computation therefore takes place after any compression -// and encryption steps, if any. -// +// +// - Type: the type of the page: indicates which of the *_header fields is set * +// +// - UncompressedPageSize: Uncompressed page size in bytes (not including this header) * +// +// - CompressedPageSize: Compressed (and potentially encrypted) page size in bytes, not including this header * +// +// - Crc: The 32-bit CRC checksum for the page, to be be calculated as follows: +// +// - The standard CRC32 algorithm is used (with polynomial 0x04C11DB7, +// the same as in e.g. GZip). +// +// - All page types can have a CRC (v1 and v2 data pages, dictionary pages, +// etc.). +// +// - The CRC is computed on the serialization binary representation of the page +// (as written to disk), excluding the page header. For example, for v1 +// data pages, the CRC is computed on the concatenation of repetition levels, +// definition levels and column values (optionally compressed, optionally +// encrypted). +// +// - The CRC computation therefore takes place after any compression +// and encryption steps, if any. +// // If enabled, this allows for disabling checksumming in HDFS if only a few // pages need to be read. -// - DataPageHeader -// - IndexPageHeader -// - DictionaryPageHeader -// - DataPageHeaderV2 +// - DataPageHeader +// - IndexPageHeader +// - DictionaryPageHeader +// - DataPageHeaderV2 type PageHeader struct { - Type PageType `thrift:"type,1,required" db:"type" json:"type"` - UncompressedPageSize int32 `thrift:"uncompressed_page_size,2,required" db:"uncompressed_page_size" json:"uncompressed_page_size"` - CompressedPageSize int32 `thrift:"compressed_page_size,3,required" db:"compressed_page_size" json:"compressed_page_size"` - Crc *int32 `thrift:"crc,4" db:"crc" json:"crc,omitempty"` - DataPageHeader *DataPageHeader `thrift:"data_page_header,5" db:"data_page_header" json:"data_page_header,omitempty"` - IndexPageHeader *IndexPageHeader `thrift:"index_page_header,6" db:"index_page_header" json:"index_page_header,omitempty"` - DictionaryPageHeader *DictionaryPageHeader `thrift:"dictionary_page_header,7" db:"dictionary_page_header" json:"dictionary_page_header,omitempty"` - DataPageHeaderV2 *DataPageHeaderV2 `thrift:"data_page_header_v2,8" db:"data_page_header_v2" json:"data_page_header_v2,omitempty"` + Type PageType `thrift:"type,1,required" db:"type" json:"type"` + UncompressedPageSize int32 `thrift:"uncompressed_page_size,2,required" db:"uncompressed_page_size" json:"uncompressed_page_size"` + CompressedPageSize int32 `thrift:"compressed_page_size,3,required" db:"compressed_page_size" json:"compressed_page_size"` + Crc *int32 `thrift:"crc,4" db:"crc" json:"crc,omitempty"` + DataPageHeader *DataPageHeader `thrift:"data_page_header,5" db:"data_page_header" json:"data_page_header,omitempty"` + IndexPageHeader *IndexPageHeader `thrift:"index_page_header,6" db:"index_page_header" json:"index_page_header,omitempty"` + DictionaryPageHeader *DictionaryPageHeader `thrift:"dictionary_page_header,7" db:"dictionary_page_header" json:"dictionary_page_header,omitempty"` + DataPageHeaderV2 *DataPageHeaderV2 `thrift:"data_page_header_v2,8" db:"data_page_header_v2" json:"data_page_header_v2,omitempty"` } func NewPageHeader() *PageHeader { - return &PageHeader{} + return &PageHeader{} } - func (p *PageHeader) GetType() PageType { - return p.Type + return p.Type } func (p *PageHeader) GetUncompressedPageSize() int32 { - return p.UncompressedPageSize + return p.UncompressedPageSize } func (p *PageHeader) GetCompressedPageSize() int32 { - return p.CompressedPageSize + return p.CompressedPageSize } + var PageHeader_Crc_DEFAULT int32 + func (p *PageHeader) GetCrc() int32 { - if !p.IsSetCrc() { - return PageHeader_Crc_DEFAULT - } -return *p.Crc + if !p.IsSetCrc() { + return PageHeader_Crc_DEFAULT + } + return *p.Crc } + var PageHeader_DataPageHeader_DEFAULT *DataPageHeader + func (p *PageHeader) GetDataPageHeader() *DataPageHeader { - if !p.IsSetDataPageHeader() { - return PageHeader_DataPageHeader_DEFAULT - } -return p.DataPageHeader + if !p.IsSetDataPageHeader() { + return PageHeader_DataPageHeader_DEFAULT + } + return p.DataPageHeader } + var PageHeader_IndexPageHeader_DEFAULT *IndexPageHeader + func (p *PageHeader) GetIndexPageHeader() *IndexPageHeader { - if !p.IsSetIndexPageHeader() { - return PageHeader_IndexPageHeader_DEFAULT - } -return p.IndexPageHeader + if !p.IsSetIndexPageHeader() { + return PageHeader_IndexPageHeader_DEFAULT + } + return p.IndexPageHeader } + var PageHeader_DictionaryPageHeader_DEFAULT *DictionaryPageHeader + func (p *PageHeader) GetDictionaryPageHeader() *DictionaryPageHeader { - if !p.IsSetDictionaryPageHeader() { - return PageHeader_DictionaryPageHeader_DEFAULT - } -return p.DictionaryPageHeader + if !p.IsSetDictionaryPageHeader() { + return PageHeader_DictionaryPageHeader_DEFAULT + } + return p.DictionaryPageHeader } + var PageHeader_DataPageHeaderV2_DEFAULT *DataPageHeaderV2 + func (p *PageHeader) GetDataPageHeaderV2() *DataPageHeaderV2 { - if !p.IsSetDataPageHeaderV2() { - return PageHeader_DataPageHeaderV2_DEFAULT - } -return p.DataPageHeaderV2 + if !p.IsSetDataPageHeaderV2() { + return PageHeader_DataPageHeaderV2_DEFAULT + } + return p.DataPageHeaderV2 } func (p *PageHeader) IsSetCrc() bool { - return p.Crc != nil + return p.Crc != nil } func (p *PageHeader) IsSetDataPageHeader() bool { - return p.DataPageHeader != nil + return p.DataPageHeader != nil } func (p *PageHeader) IsSetIndexPageHeader() bool { - return p.IndexPageHeader != nil + return p.IndexPageHeader != nil } func (p *PageHeader) IsSetDictionaryPageHeader() bool { - return p.DictionaryPageHeader != nil + return p.DictionaryPageHeader != nil } func (p *PageHeader) IsSetDataPageHeaderV2() bool { - return p.DataPageHeaderV2 != nil + return p.DataPageHeaderV2 != nil } func (p *PageHeader) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - var issetType bool = false; - var issetUncompressedPageSize bool = false; - var issetCompressedPageSize bool = false; - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.I32 { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - issetType = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.I32 { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - issetUncompressedPageSize = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 3: - if fieldTypeId == thrift.I32 { - if err := p.ReadField3(ctx, iprot); err != nil { - return err - } - issetCompressedPageSize = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 4: - if fieldTypeId == thrift.I32 { - if err := p.ReadField4(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 5: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField5(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 6: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField6(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 7: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField7(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 8: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField8(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - if !issetType{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Type is not set")); - } - if !issetUncompressedPageSize{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field UncompressedPageSize is not set")); - } - if !issetCompressedPageSize{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field CompressedPageSize is not set")); - } - return nil -} - -func (p *PageHeader) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 1: ", err) -} else { - temp := PageType(v) - p.Type = temp -} - return nil -} - -func (p *PageHeader) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 2: ", err) -} else { - p.UncompressedPageSize = v -} - return nil -} - -func (p *PageHeader) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 3: ", err) -} else { - p.CompressedPageSize = v -} - return nil -} - -func (p *PageHeader) ReadField4(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 4: ", err) -} else { - p.Crc = &v -} - return nil -} - -func (p *PageHeader) ReadField5(ctx context.Context, iprot thrift.TProtocol) error { - p.DataPageHeader = &DataPageHeader{} - if err := p.DataPageHeader.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.DataPageHeader), err) - } - return nil -} - -func (p *PageHeader) ReadField6(ctx context.Context, iprot thrift.TProtocol) error { - p.IndexPageHeader = &IndexPageHeader{} - if err := p.IndexPageHeader.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.IndexPageHeader), err) - } - return nil -} - -func (p *PageHeader) ReadField7(ctx context.Context, iprot thrift.TProtocol) error { - p.DictionaryPageHeader = &DictionaryPageHeader{} - if err := p.DictionaryPageHeader.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.DictionaryPageHeader), err) - } - return nil -} - -func (p *PageHeader) ReadField8(ctx context.Context, iprot thrift.TProtocol) error { - p.DataPageHeaderV2 = &DataPageHeaderV2{ - IsCompressed: true, -} - if err := p.DataPageHeaderV2.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.DataPageHeaderV2), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + var issetType bool = false + var issetUncompressedPageSize bool = false + var issetCompressedPageSize bool = false + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.I32 { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + issetType = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.I32 { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + issetUncompressedPageSize = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 3: + if fieldTypeId == thrift.I32 { + if err := p.ReadField3(ctx, iprot); err != nil { + return err + } + issetCompressedPageSize = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 4: + if fieldTypeId == thrift.I32 { + if err := p.ReadField4(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 5: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField5(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 6: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField6(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 7: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField7(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 8: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField8(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + if !issetType { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Type is not set")) + } + if !issetUncompressedPageSize { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field UncompressedPageSize is not set")) + } + if !issetCompressedPageSize { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field CompressedPageSize is not set")) + } + return nil +} + +func (p *PageHeader) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 1: ", err) + } else { + temp := PageType(v) + p.Type = temp + } + return nil +} + +func (p *PageHeader) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 2: ", err) + } else { + p.UncompressedPageSize = v + } + return nil +} + +func (p *PageHeader) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 3: ", err) + } else { + p.CompressedPageSize = v + } + return nil +} + +func (p *PageHeader) ReadField4(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 4: ", err) + } else { + p.Crc = &v + } + return nil +} + +func (p *PageHeader) ReadField5(ctx context.Context, iprot thrift.TProtocol) error { + p.DataPageHeader = &DataPageHeader{} + if err := p.DataPageHeader.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.DataPageHeader), err) + } + return nil +} + +func (p *PageHeader) ReadField6(ctx context.Context, iprot thrift.TProtocol) error { + p.IndexPageHeader = &IndexPageHeader{} + if err := p.IndexPageHeader.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.IndexPageHeader), err) + } + return nil +} + +func (p *PageHeader) ReadField7(ctx context.Context, iprot thrift.TProtocol) error { + p.DictionaryPageHeader = &DictionaryPageHeader{} + if err := p.DictionaryPageHeader.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.DictionaryPageHeader), err) + } + return nil +} + +func (p *PageHeader) ReadField8(ctx context.Context, iprot thrift.TProtocol) error { + p.DataPageHeaderV2 = &DataPageHeaderV2{ + IsCompressed: true, + } + if err := p.DataPageHeaderV2.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.DataPageHeaderV2), err) + } + return nil } func (p *PageHeader) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "PageHeader"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - if err := p.writeField3(ctx, oprot); err != nil { return err } - if err := p.writeField4(ctx, oprot); err != nil { return err } - if err := p.writeField5(ctx, oprot); err != nil { return err } - if err := p.writeField6(ctx, oprot); err != nil { return err } - if err := p.writeField7(ctx, oprot); err != nil { return err } - if err := p.writeField8(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "PageHeader"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + if err := p.writeField3(ctx, oprot); err != nil { + return err + } + if err := p.writeField4(ctx, oprot); err != nil { + return err + } + if err := p.writeField5(ctx, oprot); err != nil { + return err + } + if err := p.writeField6(ctx, oprot); err != nil { + return err + } + if err := p.writeField7(ctx, oprot); err != nil { + return err + } + if err := p.writeField8(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *PageHeader) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "type", thrift.I32, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:type: ", p), err) } - if err := oprot.WriteI32(ctx, int32(p.Type)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.type (1) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:type: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "type", thrift.I32, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:type: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.Type)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.type (1) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:type: ", p), err) + } + return err } func (p *PageHeader) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "uncompressed_page_size", thrift.I32, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:uncompressed_page_size: ", p), err) } - if err := oprot.WriteI32(ctx, int32(p.UncompressedPageSize)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.uncompressed_page_size (2) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:uncompressed_page_size: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "uncompressed_page_size", thrift.I32, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:uncompressed_page_size: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.UncompressedPageSize)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.uncompressed_page_size (2) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:uncompressed_page_size: ", p), err) + } + return err } func (p *PageHeader) writeField3(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "compressed_page_size", thrift.I32, 3); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:compressed_page_size: ", p), err) } - if err := oprot.WriteI32(ctx, int32(p.CompressedPageSize)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.compressed_page_size (3) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 3:compressed_page_size: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "compressed_page_size", thrift.I32, 3); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:compressed_page_size: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.CompressedPageSize)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.compressed_page_size (3) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 3:compressed_page_size: ", p), err) + } + return err } func (p *PageHeader) writeField4(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetCrc() { - if err := oprot.WriteFieldBegin(ctx, "crc", thrift.I32, 4); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:crc: ", p), err) } - if err := oprot.WriteI32(ctx, int32(*p.Crc)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.crc (4) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 4:crc: ", p), err) } - } - return err + if p.IsSetCrc() { + if err := oprot.WriteFieldBegin(ctx, "crc", thrift.I32, 4); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:crc: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(*p.Crc)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.crc (4) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 4:crc: ", p), err) + } + } + return err } func (p *PageHeader) writeField5(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetDataPageHeader() { - if err := oprot.WriteFieldBegin(ctx, "data_page_header", thrift.STRUCT, 5); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:data_page_header: ", p), err) } - if err := p.DataPageHeader.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.DataPageHeader), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 5:data_page_header: ", p), err) } - } - return err + if p.IsSetDataPageHeader() { + if err := oprot.WriteFieldBegin(ctx, "data_page_header", thrift.STRUCT, 5); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:data_page_header: ", p), err) + } + if err := p.DataPageHeader.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.DataPageHeader), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 5:data_page_header: ", p), err) + } + } + return err } func (p *PageHeader) writeField6(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetIndexPageHeader() { - if err := oprot.WriteFieldBegin(ctx, "index_page_header", thrift.STRUCT, 6); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 6:index_page_header: ", p), err) } - if err := p.IndexPageHeader.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.IndexPageHeader), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 6:index_page_header: ", p), err) } - } - return err + if p.IsSetIndexPageHeader() { + if err := oprot.WriteFieldBegin(ctx, "index_page_header", thrift.STRUCT, 6); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 6:index_page_header: ", p), err) + } + if err := p.IndexPageHeader.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.IndexPageHeader), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 6:index_page_header: ", p), err) + } + } + return err } func (p *PageHeader) writeField7(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetDictionaryPageHeader() { - if err := oprot.WriteFieldBegin(ctx, "dictionary_page_header", thrift.STRUCT, 7); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 7:dictionary_page_header: ", p), err) } - if err := p.DictionaryPageHeader.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.DictionaryPageHeader), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 7:dictionary_page_header: ", p), err) } - } - return err + if p.IsSetDictionaryPageHeader() { + if err := oprot.WriteFieldBegin(ctx, "dictionary_page_header", thrift.STRUCT, 7); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 7:dictionary_page_header: ", p), err) + } + if err := p.DictionaryPageHeader.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.DictionaryPageHeader), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 7:dictionary_page_header: ", p), err) + } + } + return err } func (p *PageHeader) writeField8(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetDataPageHeaderV2() { - if err := oprot.WriteFieldBegin(ctx, "data_page_header_v2", thrift.STRUCT, 8); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 8:data_page_header_v2: ", p), err) } - if err := p.DataPageHeaderV2.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.DataPageHeaderV2), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 8:data_page_header_v2: ", p), err) } - } - return err + if p.IsSetDataPageHeaderV2() { + if err := oprot.WriteFieldBegin(ctx, "data_page_header_v2", thrift.STRUCT, 8); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 8:data_page_header_v2: ", p), err) + } + if err := p.DataPageHeaderV2.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.DataPageHeaderV2), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 8:data_page_header_v2: ", p), err) + } + } + return err } func (p *PageHeader) Equals(other *PageHeader) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if p.Type != other.Type { return false } - if p.UncompressedPageSize != other.UncompressedPageSize { return false } - if p.CompressedPageSize != other.CompressedPageSize { return false } - if p.Crc != other.Crc { - if p.Crc == nil || other.Crc == nil { - return false - } - if (*p.Crc) != (*other.Crc) { return false } - } - if !p.DataPageHeader.Equals(other.DataPageHeader) { return false } - if !p.IndexPageHeader.Equals(other.IndexPageHeader) { return false } - if !p.DictionaryPageHeader.Equals(other.DictionaryPageHeader) { return false } - if !p.DataPageHeaderV2.Equals(other.DataPageHeaderV2) { return false } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if p.Type != other.Type { + return false + } + if p.UncompressedPageSize != other.UncompressedPageSize { + return false + } + if p.CompressedPageSize != other.CompressedPageSize { + return false + } + if p.Crc != other.Crc { + if p.Crc == nil || other.Crc == nil { + return false + } + if (*p.Crc) != (*other.Crc) { + return false + } + } + if !p.DataPageHeader.Equals(other.DataPageHeader) { + return false + } + if !p.IndexPageHeader.Equals(other.IndexPageHeader) { + return false + } + if !p.DictionaryPageHeader.Equals(other.DictionaryPageHeader) { + return false + } + if !p.DataPageHeaderV2.Equals(other.DataPageHeaderV2) { + return false + } + return true } func (p *PageHeader) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("PageHeader(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("PageHeader(%+v)", *p) } func (p *PageHeader) Validate() error { - return nil + return nil } + // Wrapper struct to store key values -// +// // Attributes: -// - Key -// - Value +// - Key +// - Value type KeyValue struct { - Key string `thrift:"key,1,required" db:"key" json:"key"` - Value *string `thrift:"value,2" db:"value" json:"value,omitempty"` + Key string `thrift:"key,1,required" db:"key" json:"key"` + Value *string `thrift:"value,2" db:"value" json:"value,omitempty"` } func NewKeyValue() *KeyValue { - return &KeyValue{} + return &KeyValue{} } - func (p *KeyValue) GetKey() string { - return p.Key + return p.Key } + var KeyValue_Value_DEFAULT string + func (p *KeyValue) GetValue() string { - if !p.IsSetValue() { - return KeyValue_Value_DEFAULT - } -return *p.Value + if !p.IsSetValue() { + return KeyValue_Value_DEFAULT + } + return *p.Value } func (p *KeyValue) IsSetValue() bool { - return p.Value != nil + return p.Value != nil } func (p *KeyValue) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - var issetKey bool = false; - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.STRING { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - issetKey = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.STRING { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - if !issetKey{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Key is not set")); - } - return nil -} - -func (p *KeyValue) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadString(ctx); err != nil { - return thrift.PrependError("error reading field 1: ", err) -} else { - p.Key = v -} - return nil -} - -func (p *KeyValue) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadString(ctx); err != nil { - return thrift.PrependError("error reading field 2: ", err) -} else { - p.Value = &v -} - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + var issetKey bool = false + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.STRING { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + issetKey = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.STRING { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + if !issetKey { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Key is not set")) + } + return nil +} + +func (p *KeyValue) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadString(ctx); err != nil { + return thrift.PrependError("error reading field 1: ", err) + } else { + p.Key = v + } + return nil +} + +func (p *KeyValue) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadString(ctx); err != nil { + return thrift.PrependError("error reading field 2: ", err) + } else { + p.Value = &v + } + return nil } func (p *KeyValue) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "KeyValue"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "KeyValue"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *KeyValue) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "key", thrift.STRING, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:key: ", p), err) } - if err := oprot.WriteString(ctx, string(p.Key)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.key (1) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:key: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "key", thrift.STRING, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:key: ", p), err) + } + if err := oprot.WriteString(ctx, string(p.Key)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.key (1) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:key: ", p), err) + } + return err } func (p *KeyValue) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetValue() { - if err := oprot.WriteFieldBegin(ctx, "value", thrift.STRING, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:value: ", p), err) } - if err := oprot.WriteString(ctx, string(*p.Value)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.value (2) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:value: ", p), err) } - } - return err + if p.IsSetValue() { + if err := oprot.WriteFieldBegin(ctx, "value", thrift.STRING, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:value: ", p), err) + } + if err := oprot.WriteString(ctx, string(*p.Value)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.value (2) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:value: ", p), err) + } + } + return err } func (p *KeyValue) Equals(other *KeyValue) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if p.Key != other.Key { return false } - if p.Value != other.Value { - if p.Value == nil || other.Value == nil { - return false - } - if (*p.Value) != (*other.Value) { return false } - } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if p.Key != other.Key { + return false + } + if p.Value != other.Value { + if p.Value == nil || other.Value == nil { + return false + } + if (*p.Value) != (*other.Value) { + return false + } + } + return true } func (p *KeyValue) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("KeyValue(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("KeyValue(%+v)", *p) } func (p *KeyValue) Validate() error { - return nil + return nil } + // Wrapper struct to specify sort order -// +// // Attributes: -// - ColumnIdx: The column index (in this row group) * -// - Descending: If true, indicates this column is sorted in descending order. * -// - NullsFirst: If true, nulls will come before non-null values, otherwise, +// - ColumnIdx: The column index (in this row group) * +// - Descending: If true, indicates this column is sorted in descending order. * +// - NullsFirst: If true, nulls will come before non-null values, otherwise, +// // nulls go at the end. type SortingColumn struct { - ColumnIdx int32 `thrift:"column_idx,1,required" db:"column_idx" json:"column_idx"` - Descending bool `thrift:"descending,2,required" db:"descending" json:"descending"` - NullsFirst bool `thrift:"nulls_first,3,required" db:"nulls_first" json:"nulls_first"` + ColumnIdx int32 `thrift:"column_idx,1,required" db:"column_idx" json:"column_idx"` + Descending bool `thrift:"descending,2,required" db:"descending" json:"descending"` + NullsFirst bool `thrift:"nulls_first,3,required" db:"nulls_first" json:"nulls_first"` } func NewSortingColumn() *SortingColumn { - return &SortingColumn{} + return &SortingColumn{} } - func (p *SortingColumn) GetColumnIdx() int32 { - return p.ColumnIdx + return p.ColumnIdx } func (p *SortingColumn) GetDescending() bool { - return p.Descending + return p.Descending } func (p *SortingColumn) GetNullsFirst() bool { - return p.NullsFirst + return p.NullsFirst } func (p *SortingColumn) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - var issetColumnIdx bool = false; - var issetDescending bool = false; - var issetNullsFirst bool = false; - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.I32 { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - issetColumnIdx = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.BOOL { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - issetDescending = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 3: - if fieldTypeId == thrift.BOOL { - if err := p.ReadField3(ctx, iprot); err != nil { - return err - } - issetNullsFirst = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - if !issetColumnIdx{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field ColumnIdx is not set")); - } - if !issetDescending{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Descending is not set")); - } - if !issetNullsFirst{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field NullsFirst is not set")); - } - return nil -} - -func (p *SortingColumn) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 1: ", err) -} else { - p.ColumnIdx = v -} - return nil -} - -func (p *SortingColumn) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadBool(ctx); err != nil { - return thrift.PrependError("error reading field 2: ", err) -} else { - p.Descending = v -} - return nil -} - -func (p *SortingColumn) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadBool(ctx); err != nil { - return thrift.PrependError("error reading field 3: ", err) -} else { - p.NullsFirst = v -} - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + var issetColumnIdx bool = false + var issetDescending bool = false + var issetNullsFirst bool = false + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.I32 { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + issetColumnIdx = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.BOOL { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + issetDescending = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 3: + if fieldTypeId == thrift.BOOL { + if err := p.ReadField3(ctx, iprot); err != nil { + return err + } + issetNullsFirst = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + if !issetColumnIdx { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field ColumnIdx is not set")) + } + if !issetDescending { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Descending is not set")) + } + if !issetNullsFirst { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field NullsFirst is not set")) + } + return nil +} + +func (p *SortingColumn) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 1: ", err) + } else { + p.ColumnIdx = v + } + return nil +} + +func (p *SortingColumn) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadBool(ctx); err != nil { + return thrift.PrependError("error reading field 2: ", err) + } else { + p.Descending = v + } + return nil +} + +func (p *SortingColumn) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadBool(ctx); err != nil { + return thrift.PrependError("error reading field 3: ", err) + } else { + p.NullsFirst = v + } + return nil } func (p *SortingColumn) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "SortingColumn"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - if err := p.writeField3(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "SortingColumn"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + if err := p.writeField3(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *SortingColumn) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "column_idx", thrift.I32, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:column_idx: ", p), err) } - if err := oprot.WriteI32(ctx, int32(p.ColumnIdx)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.column_idx (1) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:column_idx: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "column_idx", thrift.I32, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:column_idx: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.ColumnIdx)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.column_idx (1) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:column_idx: ", p), err) + } + return err } func (p *SortingColumn) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "descending", thrift.BOOL, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:descending: ", p), err) } - if err := oprot.WriteBool(ctx, bool(p.Descending)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.descending (2) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:descending: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "descending", thrift.BOOL, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:descending: ", p), err) + } + if err := oprot.WriteBool(ctx, bool(p.Descending)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.descending (2) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:descending: ", p), err) + } + return err } func (p *SortingColumn) writeField3(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "nulls_first", thrift.BOOL, 3); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:nulls_first: ", p), err) } - if err := oprot.WriteBool(ctx, bool(p.NullsFirst)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.nulls_first (3) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 3:nulls_first: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "nulls_first", thrift.BOOL, 3); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:nulls_first: ", p), err) + } + if err := oprot.WriteBool(ctx, bool(p.NullsFirst)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.nulls_first (3) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 3:nulls_first: ", p), err) + } + return err } func (p *SortingColumn) Equals(other *SortingColumn) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if p.ColumnIdx != other.ColumnIdx { return false } - if p.Descending != other.Descending { return false } - if p.NullsFirst != other.NullsFirst { return false } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if p.ColumnIdx != other.ColumnIdx { + return false + } + if p.Descending != other.Descending { + return false + } + if p.NullsFirst != other.NullsFirst { + return false + } + return true } func (p *SortingColumn) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("SortingColumn(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("SortingColumn(%+v)", *p) } func (p *SortingColumn) Validate() error { - return nil + return nil } + // statistics of a given page type and encoding -// +// // Attributes: -// - PageType: the page type (data/dic/...) * -// - Encoding: encoding of the page * -// - Count: number of pages of this type with this encoding * +// - PageType: the page type (data/dic/...) * +// - Encoding: encoding of the page * +// - Count: number of pages of this type with this encoding * type PageEncodingStats struct { - PageType PageType `thrift:"page_type,1,required" db:"page_type" json:"page_type"` - Encoding Encoding `thrift:"encoding,2,required" db:"encoding" json:"encoding"` - Count int32 `thrift:"count,3,required" db:"count" json:"count"` + PageType PageType `thrift:"page_type,1,required" db:"page_type" json:"page_type"` + Encoding Encoding `thrift:"encoding,2,required" db:"encoding" json:"encoding"` + Count int32 `thrift:"count,3,required" db:"count" json:"count"` } func NewPageEncodingStats() *PageEncodingStats { - return &PageEncodingStats{} + return &PageEncodingStats{} } - func (p *PageEncodingStats) GetPageType() PageType { - return p.PageType + return p.PageType } func (p *PageEncodingStats) GetEncoding() Encoding { - return p.Encoding + return p.Encoding } func (p *PageEncodingStats) GetCount() int32 { - return p.Count + return p.Count } func (p *PageEncodingStats) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - var issetPageType bool = false; - var issetEncoding bool = false; - var issetCount bool = false; - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.I32 { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - issetPageType = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.I32 { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - issetEncoding = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 3: - if fieldTypeId == thrift.I32 { - if err := p.ReadField3(ctx, iprot); err != nil { - return err - } - issetCount = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - if !issetPageType{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field PageType is not set")); - } - if !issetEncoding{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Encoding is not set")); - } - if !issetCount{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Count is not set")); - } - return nil -} - -func (p *PageEncodingStats) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 1: ", err) -} else { - temp := PageType(v) - p.PageType = temp -} - return nil -} - -func (p *PageEncodingStats) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 2: ", err) -} else { - temp := Encoding(v) - p.Encoding = temp -} - return nil -} - -func (p *PageEncodingStats) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 3: ", err) -} else { - p.Count = v -} - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + var issetPageType bool = false + var issetEncoding bool = false + var issetCount bool = false + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.I32 { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + issetPageType = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.I32 { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + issetEncoding = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 3: + if fieldTypeId == thrift.I32 { + if err := p.ReadField3(ctx, iprot); err != nil { + return err + } + issetCount = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + if !issetPageType { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field PageType is not set")) + } + if !issetEncoding { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Encoding is not set")) + } + if !issetCount { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Count is not set")) + } + return nil +} + +func (p *PageEncodingStats) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 1: ", err) + } else { + temp := PageType(v) + p.PageType = temp + } + return nil +} + +func (p *PageEncodingStats) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 2: ", err) + } else { + temp := Encoding(v) + p.Encoding = temp + } + return nil +} + +func (p *PageEncodingStats) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 3: ", err) + } else { + p.Count = v + } + return nil } func (p *PageEncodingStats) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "PageEncodingStats"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - if err := p.writeField3(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "PageEncodingStats"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + if err := p.writeField3(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *PageEncodingStats) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "page_type", thrift.I32, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:page_type: ", p), err) } - if err := oprot.WriteI32(ctx, int32(p.PageType)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.page_type (1) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:page_type: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "page_type", thrift.I32, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:page_type: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.PageType)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.page_type (1) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:page_type: ", p), err) + } + return err } func (p *PageEncodingStats) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "encoding", thrift.I32, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:encoding: ", p), err) } - if err := oprot.WriteI32(ctx, int32(p.Encoding)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.encoding (2) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:encoding: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "encoding", thrift.I32, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:encoding: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.Encoding)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.encoding (2) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:encoding: ", p), err) + } + return err } func (p *PageEncodingStats) writeField3(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "count", thrift.I32, 3); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:count: ", p), err) } - if err := oprot.WriteI32(ctx, int32(p.Count)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.count (3) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 3:count: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "count", thrift.I32, 3); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:count: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.Count)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.count (3) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 3:count: ", p), err) + } + return err } func (p *PageEncodingStats) Equals(other *PageEncodingStats) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if p.PageType != other.PageType { return false } - if p.Encoding != other.Encoding { return false } - if p.Count != other.Count { return false } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if p.PageType != other.PageType { + return false + } + if p.Encoding != other.Encoding { + return false + } + if p.Count != other.Count { + return false + } + return true } func (p *PageEncodingStats) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("PageEncodingStats(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("PageEncodingStats(%+v)", *p) } func (p *PageEncodingStats) Validate() error { - return nil + return nil } + // Description for column metadata -// +// // Attributes: -// - Type: Type of this column * -// - Encodings: Set of all encodings used for this column. The purpose is to validate +// - Type: Type of this column * +// - Encodings: Set of all encodings used for this column. The purpose is to validate +// // whether we can decode those pages. * -// - PathInSchema: Path in schema * -// - Codec: Compression codec * -// - NumValues: Number of values in this column * -// - TotalUncompressedSize: total byte size of all uncompressed pages in this column chunk (including the headers) * -// - TotalCompressedSize: total byte size of all compressed, and potentially encrypted, pages +// - PathInSchema: Path in schema * +// - Codec: Compression codec * +// - NumValues: Number of values in this column * +// - TotalUncompressedSize: total byte size of all uncompressed pages in this column chunk (including the headers) * +// - TotalCompressedSize: total byte size of all compressed, and potentially encrypted, pages +// // in this column chunk (including the headers) * -// - KeyValueMetadata: Optional key/value metadata * -// - DataPageOffset: Byte offset from beginning of file to first data page * -// - IndexPageOffset: Byte offset from beginning of file to root index page * -// - DictionaryPageOffset: Byte offset from the beginning of file to first (only) dictionary page * -// - Statistics: optional statistics for this column chunk -// - EncodingStats: Set of all encodings used for pages in this column chunk. +// - KeyValueMetadata: Optional key/value metadata * +// - DataPageOffset: Byte offset from beginning of file to first data page * +// - IndexPageOffset: Byte offset from beginning of file to root index page * +// - DictionaryPageOffset: Byte offset from the beginning of file to first (only) dictionary page * +// - Statistics: optional statistics for this column chunk +// - EncodingStats: Set of all encodings used for pages in this column chunk. +// // This information can be used to determine if all data pages are // dictionary encoded for example * -// - BloomFilterOffset: Byte offset from beginning of file to Bloom filter data. * +// - BloomFilterOffset: Byte offset from beginning of file to Bloom filter data. * type ColumnMetaData struct { - Type Type `thrift:"type,1,required" db:"type" json:"type"` - Encodings []Encoding `thrift:"encodings,2,required" db:"encodings" json:"encodings"` - PathInSchema []string `thrift:"path_in_schema,3,required" db:"path_in_schema" json:"path_in_schema"` - Codec CompressionCodec `thrift:"codec,4,required" db:"codec" json:"codec"` - NumValues int64 `thrift:"num_values,5,required" db:"num_values" json:"num_values"` - TotalUncompressedSize int64 `thrift:"total_uncompressed_size,6,required" db:"total_uncompressed_size" json:"total_uncompressed_size"` - TotalCompressedSize int64 `thrift:"total_compressed_size,7,required" db:"total_compressed_size" json:"total_compressed_size"` - KeyValueMetadata []*KeyValue `thrift:"key_value_metadata,8" db:"key_value_metadata" json:"key_value_metadata,omitempty"` - DataPageOffset int64 `thrift:"data_page_offset,9,required" db:"data_page_offset" json:"data_page_offset"` - IndexPageOffset *int64 `thrift:"index_page_offset,10" db:"index_page_offset" json:"index_page_offset,omitempty"` - DictionaryPageOffset *int64 `thrift:"dictionary_page_offset,11" db:"dictionary_page_offset" json:"dictionary_page_offset,omitempty"` - Statistics *Statistics `thrift:"statistics,12" db:"statistics" json:"statistics,omitempty"` - EncodingStats []*PageEncodingStats `thrift:"encoding_stats,13" db:"encoding_stats" json:"encoding_stats,omitempty"` - BloomFilterOffset *int64 `thrift:"bloom_filter_offset,14" db:"bloom_filter_offset" json:"bloom_filter_offset,omitempty"` + Type Type `thrift:"type,1,required" db:"type" json:"type"` + Encodings []Encoding `thrift:"encodings,2,required" db:"encodings" json:"encodings"` + PathInSchema []string `thrift:"path_in_schema,3,required" db:"path_in_schema" json:"path_in_schema"` + Codec CompressionCodec `thrift:"codec,4,required" db:"codec" json:"codec"` + NumValues int64 `thrift:"num_values,5,required" db:"num_values" json:"num_values"` + TotalUncompressedSize int64 `thrift:"total_uncompressed_size,6,required" db:"total_uncompressed_size" json:"total_uncompressed_size"` + TotalCompressedSize int64 `thrift:"total_compressed_size,7,required" db:"total_compressed_size" json:"total_compressed_size"` + KeyValueMetadata []*KeyValue `thrift:"key_value_metadata,8" db:"key_value_metadata" json:"key_value_metadata,omitempty"` + DataPageOffset int64 `thrift:"data_page_offset,9,required" db:"data_page_offset" json:"data_page_offset"` + IndexPageOffset *int64 `thrift:"index_page_offset,10" db:"index_page_offset" json:"index_page_offset,omitempty"` + DictionaryPageOffset *int64 `thrift:"dictionary_page_offset,11" db:"dictionary_page_offset" json:"dictionary_page_offset,omitempty"` + Statistics *Statistics `thrift:"statistics,12" db:"statistics" json:"statistics,omitempty"` + EncodingStats []*PageEncodingStats `thrift:"encoding_stats,13" db:"encoding_stats" json:"encoding_stats,omitempty"` + BloomFilterOffset *int64 `thrift:"bloom_filter_offset,14" db:"bloom_filter_offset" json:"bloom_filter_offset,omitempty"` } func NewColumnMetaData() *ColumnMetaData { - return &ColumnMetaData{} + return &ColumnMetaData{} } - func (p *ColumnMetaData) GetType() Type { - return p.Type + return p.Type } func (p *ColumnMetaData) GetEncodings() []Encoding { - return p.Encodings + return p.Encodings } func (p *ColumnMetaData) GetPathInSchema() []string { - return p.PathInSchema + return p.PathInSchema } func (p *ColumnMetaData) GetCodec() CompressionCodec { - return p.Codec + return p.Codec } func (p *ColumnMetaData) GetNumValues() int64 { - return p.NumValues + return p.NumValues } func (p *ColumnMetaData) GetTotalUncompressedSize() int64 { - return p.TotalUncompressedSize + return p.TotalUncompressedSize } func (p *ColumnMetaData) GetTotalCompressedSize() int64 { - return p.TotalCompressedSize + return p.TotalCompressedSize } + var ColumnMetaData_KeyValueMetadata_DEFAULT []*KeyValue func (p *ColumnMetaData) GetKeyValueMetadata() []*KeyValue { - return p.KeyValueMetadata + return p.KeyValueMetadata } func (p *ColumnMetaData) GetDataPageOffset() int64 { - return p.DataPageOffset + return p.DataPageOffset } + var ColumnMetaData_IndexPageOffset_DEFAULT int64 + func (p *ColumnMetaData) GetIndexPageOffset() int64 { - if !p.IsSetIndexPageOffset() { - return ColumnMetaData_IndexPageOffset_DEFAULT - } -return *p.IndexPageOffset + if !p.IsSetIndexPageOffset() { + return ColumnMetaData_IndexPageOffset_DEFAULT + } + return *p.IndexPageOffset } + var ColumnMetaData_DictionaryPageOffset_DEFAULT int64 + func (p *ColumnMetaData) GetDictionaryPageOffset() int64 { - if !p.IsSetDictionaryPageOffset() { - return ColumnMetaData_DictionaryPageOffset_DEFAULT - } -return *p.DictionaryPageOffset + if !p.IsSetDictionaryPageOffset() { + return ColumnMetaData_DictionaryPageOffset_DEFAULT + } + return *p.DictionaryPageOffset } + var ColumnMetaData_Statistics_DEFAULT *Statistics + func (p *ColumnMetaData) GetStatistics() *Statistics { - if !p.IsSetStatistics() { - return ColumnMetaData_Statistics_DEFAULT - } -return p.Statistics + if !p.IsSetStatistics() { + return ColumnMetaData_Statistics_DEFAULT + } + return p.Statistics } + var ColumnMetaData_EncodingStats_DEFAULT []*PageEncodingStats func (p *ColumnMetaData) GetEncodingStats() []*PageEncodingStats { - return p.EncodingStats + return p.EncodingStats } + var ColumnMetaData_BloomFilterOffset_DEFAULT int64 + func (p *ColumnMetaData) GetBloomFilterOffset() int64 { - if !p.IsSetBloomFilterOffset() { - return ColumnMetaData_BloomFilterOffset_DEFAULT - } -return *p.BloomFilterOffset + if !p.IsSetBloomFilterOffset() { + return ColumnMetaData_BloomFilterOffset_DEFAULT + } + return *p.BloomFilterOffset } func (p *ColumnMetaData) IsSetKeyValueMetadata() bool { - return p.KeyValueMetadata != nil + return p.KeyValueMetadata != nil } func (p *ColumnMetaData) IsSetIndexPageOffset() bool { - return p.IndexPageOffset != nil + return p.IndexPageOffset != nil } func (p *ColumnMetaData) IsSetDictionaryPageOffset() bool { - return p.DictionaryPageOffset != nil + return p.DictionaryPageOffset != nil } func (p *ColumnMetaData) IsSetStatistics() bool { - return p.Statistics != nil + return p.Statistics != nil } func (p *ColumnMetaData) IsSetEncodingStats() bool { - return p.EncodingStats != nil + return p.EncodingStats != nil } func (p *ColumnMetaData) IsSetBloomFilterOffset() bool { - return p.BloomFilterOffset != nil + return p.BloomFilterOffset != nil } func (p *ColumnMetaData) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - var issetType bool = false; - var issetEncodings bool = false; - var issetPathInSchema bool = false; - var issetCodec bool = false; - var issetNumValues bool = false; - var issetTotalUncompressedSize bool = false; - var issetTotalCompressedSize bool = false; - var issetDataPageOffset bool = false; - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.I32 { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - issetType = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.LIST { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - issetEncodings = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 3: - if fieldTypeId == thrift.LIST { - if err := p.ReadField3(ctx, iprot); err != nil { - return err - } - issetPathInSchema = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 4: - if fieldTypeId == thrift.I32 { - if err := p.ReadField4(ctx, iprot); err != nil { - return err - } - issetCodec = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 5: - if fieldTypeId == thrift.I64 { - if err := p.ReadField5(ctx, iprot); err != nil { - return err - } - issetNumValues = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 6: - if fieldTypeId == thrift.I64 { - if err := p.ReadField6(ctx, iprot); err != nil { - return err - } - issetTotalUncompressedSize = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 7: - if fieldTypeId == thrift.I64 { - if err := p.ReadField7(ctx, iprot); err != nil { - return err - } - issetTotalCompressedSize = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 8: - if fieldTypeId == thrift.LIST { - if err := p.ReadField8(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 9: - if fieldTypeId == thrift.I64 { - if err := p.ReadField9(ctx, iprot); err != nil { - return err - } - issetDataPageOffset = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 10: - if fieldTypeId == thrift.I64 { - if err := p.ReadField10(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 11: - if fieldTypeId == thrift.I64 { - if err := p.ReadField11(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 12: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField12(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 13: - if fieldTypeId == thrift.LIST { - if err := p.ReadField13(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 14: - if fieldTypeId == thrift.I64 { - if err := p.ReadField14(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - if !issetType{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Type is not set")); - } - if !issetEncodings{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Encodings is not set")); - } - if !issetPathInSchema{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field PathInSchema is not set")); - } - if !issetCodec{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Codec is not set")); - } - if !issetNumValues{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field NumValues is not set")); - } - if !issetTotalUncompressedSize{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field TotalUncompressedSize is not set")); - } - if !issetTotalCompressedSize{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field TotalCompressedSize is not set")); - } - if !issetDataPageOffset{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field DataPageOffset is not set")); - } - return nil -} - -func (p *ColumnMetaData) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 1: ", err) -} else { - temp := Type(v) - p.Type = temp -} - return nil -} - -func (p *ColumnMetaData) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - _, size, err := iprot.ReadListBegin(ctx) - if err != nil { - return thrift.PrependError("error reading list begin: ", err) - } - tSlice := make([]Encoding, 0, size) - p.Encodings = tSlice - for i := 0; i < size; i ++ { -var _elem0 Encoding - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 0: ", err) -} else { - temp := Encoding(v) - _elem0 = temp -} - p.Encodings = append(p.Encodings, _elem0) - } - if err := iprot.ReadListEnd(ctx); err != nil { - return thrift.PrependError("error reading list end: ", err) - } - return nil -} - -func (p *ColumnMetaData) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { - _, size, err := iprot.ReadListBegin(ctx) - if err != nil { - return thrift.PrependError("error reading list begin: ", err) - } - tSlice := make([]string, 0, size) - p.PathInSchema = tSlice - for i := 0; i < size; i ++ { -var _elem1 string - if v, err := iprot.ReadString(ctx); err != nil { - return thrift.PrependError("error reading field 0: ", err) -} else { - _elem1 = v -} - p.PathInSchema = append(p.PathInSchema, _elem1) - } - if err := iprot.ReadListEnd(ctx); err != nil { - return thrift.PrependError("error reading list end: ", err) - } - return nil -} - -func (p *ColumnMetaData) ReadField4(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 4: ", err) -} else { - temp := CompressionCodec(v) - p.Codec = temp -} - return nil -} - -func (p *ColumnMetaData) ReadField5(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(ctx); err != nil { - return thrift.PrependError("error reading field 5: ", err) -} else { - p.NumValues = v -} - return nil -} - -func (p *ColumnMetaData) ReadField6(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(ctx); err != nil { - return thrift.PrependError("error reading field 6: ", err) -} else { - p.TotalUncompressedSize = v -} - return nil -} - -func (p *ColumnMetaData) ReadField7(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(ctx); err != nil { - return thrift.PrependError("error reading field 7: ", err) -} else { - p.TotalCompressedSize = v -} - return nil -} - -func (p *ColumnMetaData) ReadField8(ctx context.Context, iprot thrift.TProtocol) error { - _, size, err := iprot.ReadListBegin(ctx) - if err != nil { - return thrift.PrependError("error reading list begin: ", err) - } - tSlice := make([]*KeyValue, 0, size) - p.KeyValueMetadata = tSlice - for i := 0; i < size; i ++ { - _elem2 := &KeyValue{} - if err := _elem2.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem2), err) - } - p.KeyValueMetadata = append(p.KeyValueMetadata, _elem2) - } - if err := iprot.ReadListEnd(ctx); err != nil { - return thrift.PrependError("error reading list end: ", err) - } - return nil -} - -func (p *ColumnMetaData) ReadField9(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(ctx); err != nil { - return thrift.PrependError("error reading field 9: ", err) -} else { - p.DataPageOffset = v -} - return nil -} - -func (p *ColumnMetaData) ReadField10(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(ctx); err != nil { - return thrift.PrependError("error reading field 10: ", err) -} else { - p.IndexPageOffset = &v -} - return nil -} - -func (p *ColumnMetaData) ReadField11(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(ctx); err != nil { - return thrift.PrependError("error reading field 11: ", err) -} else { - p.DictionaryPageOffset = &v -} - return nil -} - -func (p *ColumnMetaData) ReadField12(ctx context.Context, iprot thrift.TProtocol) error { - p.Statistics = &Statistics{} - if err := p.Statistics.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.Statistics), err) - } - return nil -} - -func (p *ColumnMetaData) ReadField13(ctx context.Context, iprot thrift.TProtocol) error { - _, size, err := iprot.ReadListBegin(ctx) - if err != nil { - return thrift.PrependError("error reading list begin: ", err) - } - tSlice := make([]*PageEncodingStats, 0, size) - p.EncodingStats = tSlice - for i := 0; i < size; i ++ { - _elem3 := &PageEncodingStats{} - if err := _elem3.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem3), err) - } - p.EncodingStats = append(p.EncodingStats, _elem3) - } - if err := iprot.ReadListEnd(ctx); err != nil { - return thrift.PrependError("error reading list end: ", err) - } - return nil -} - -func (p *ColumnMetaData) ReadField14(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(ctx); err != nil { - return thrift.PrependError("error reading field 14: ", err) -} else { - p.BloomFilterOffset = &v -} - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + var issetType bool = false + var issetEncodings bool = false + var issetPathInSchema bool = false + var issetCodec bool = false + var issetNumValues bool = false + var issetTotalUncompressedSize bool = false + var issetTotalCompressedSize bool = false + var issetDataPageOffset bool = false + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.I32 { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + issetType = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.LIST { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + issetEncodings = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 3: + if fieldTypeId == thrift.LIST { + if err := p.ReadField3(ctx, iprot); err != nil { + return err + } + issetPathInSchema = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 4: + if fieldTypeId == thrift.I32 { + if err := p.ReadField4(ctx, iprot); err != nil { + return err + } + issetCodec = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 5: + if fieldTypeId == thrift.I64 { + if err := p.ReadField5(ctx, iprot); err != nil { + return err + } + issetNumValues = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 6: + if fieldTypeId == thrift.I64 { + if err := p.ReadField6(ctx, iprot); err != nil { + return err + } + issetTotalUncompressedSize = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 7: + if fieldTypeId == thrift.I64 { + if err := p.ReadField7(ctx, iprot); err != nil { + return err + } + issetTotalCompressedSize = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 8: + if fieldTypeId == thrift.LIST { + if err := p.ReadField8(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 9: + if fieldTypeId == thrift.I64 { + if err := p.ReadField9(ctx, iprot); err != nil { + return err + } + issetDataPageOffset = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 10: + if fieldTypeId == thrift.I64 { + if err := p.ReadField10(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 11: + if fieldTypeId == thrift.I64 { + if err := p.ReadField11(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 12: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField12(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 13: + if fieldTypeId == thrift.LIST { + if err := p.ReadField13(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 14: + if fieldTypeId == thrift.I64 { + if err := p.ReadField14(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + if !issetType { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Type is not set")) + } + if !issetEncodings { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Encodings is not set")) + } + if !issetPathInSchema { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field PathInSchema is not set")) + } + if !issetCodec { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Codec is not set")) + } + if !issetNumValues { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field NumValues is not set")) + } + if !issetTotalUncompressedSize { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field TotalUncompressedSize is not set")) + } + if !issetTotalCompressedSize { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field TotalCompressedSize is not set")) + } + if !issetDataPageOffset { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field DataPageOffset is not set")) + } + return nil +} + +func (p *ColumnMetaData) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 1: ", err) + } else { + temp := Type(v) + p.Type = temp + } + return nil +} + +func (p *ColumnMetaData) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + _, size, err := iprot.ReadListBegin(ctx) + if err != nil { + return thrift.PrependError("error reading list begin: ", err) + } + tSlice := make([]Encoding, 0, size) + p.Encodings = tSlice + for i := 0; i < size; i++ { + var _elem0 Encoding + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 0: ", err) + } else { + temp := Encoding(v) + _elem0 = temp + } + p.Encodings = append(p.Encodings, _elem0) + } + if err := iprot.ReadListEnd(ctx); err != nil { + return thrift.PrependError("error reading list end: ", err) + } + return nil +} + +func (p *ColumnMetaData) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { + _, size, err := iprot.ReadListBegin(ctx) + if err != nil { + return thrift.PrependError("error reading list begin: ", err) + } + tSlice := make([]string, 0, size) + p.PathInSchema = tSlice + for i := 0; i < size; i++ { + var _elem1 string + if v, err := iprot.ReadString(ctx); err != nil { + return thrift.PrependError("error reading field 0: ", err) + } else { + _elem1 = v + } + p.PathInSchema = append(p.PathInSchema, _elem1) + } + if err := iprot.ReadListEnd(ctx); err != nil { + return thrift.PrependError("error reading list end: ", err) + } + return nil +} + +func (p *ColumnMetaData) ReadField4(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 4: ", err) + } else { + temp := CompressionCodec(v) + p.Codec = temp + } + return nil +} + +func (p *ColumnMetaData) ReadField5(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI64(ctx); err != nil { + return thrift.PrependError("error reading field 5: ", err) + } else { + p.NumValues = v + } + return nil +} + +func (p *ColumnMetaData) ReadField6(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI64(ctx); err != nil { + return thrift.PrependError("error reading field 6: ", err) + } else { + p.TotalUncompressedSize = v + } + return nil +} + +func (p *ColumnMetaData) ReadField7(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI64(ctx); err != nil { + return thrift.PrependError("error reading field 7: ", err) + } else { + p.TotalCompressedSize = v + } + return nil +} + +func (p *ColumnMetaData) ReadField8(ctx context.Context, iprot thrift.TProtocol) error { + _, size, err := iprot.ReadListBegin(ctx) + if err != nil { + return thrift.PrependError("error reading list begin: ", err) + } + tSlice := make([]*KeyValue, 0, size) + p.KeyValueMetadata = tSlice + for i := 0; i < size; i++ { + _elem2 := &KeyValue{} + if err := _elem2.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem2), err) + } + p.KeyValueMetadata = append(p.KeyValueMetadata, _elem2) + } + if err := iprot.ReadListEnd(ctx); err != nil { + return thrift.PrependError("error reading list end: ", err) + } + return nil +} + +func (p *ColumnMetaData) ReadField9(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI64(ctx); err != nil { + return thrift.PrependError("error reading field 9: ", err) + } else { + p.DataPageOffset = v + } + return nil +} + +func (p *ColumnMetaData) ReadField10(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI64(ctx); err != nil { + return thrift.PrependError("error reading field 10: ", err) + } else { + p.IndexPageOffset = &v + } + return nil +} + +func (p *ColumnMetaData) ReadField11(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI64(ctx); err != nil { + return thrift.PrependError("error reading field 11: ", err) + } else { + p.DictionaryPageOffset = &v + } + return nil +} + +func (p *ColumnMetaData) ReadField12(ctx context.Context, iprot thrift.TProtocol) error { + p.Statistics = &Statistics{} + if err := p.Statistics.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.Statistics), err) + } + return nil +} + +func (p *ColumnMetaData) ReadField13(ctx context.Context, iprot thrift.TProtocol) error { + _, size, err := iprot.ReadListBegin(ctx) + if err != nil { + return thrift.PrependError("error reading list begin: ", err) + } + tSlice := make([]*PageEncodingStats, 0, size) + p.EncodingStats = tSlice + for i := 0; i < size; i++ { + _elem3 := &PageEncodingStats{} + if err := _elem3.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem3), err) + } + p.EncodingStats = append(p.EncodingStats, _elem3) + } + if err := iprot.ReadListEnd(ctx); err != nil { + return thrift.PrependError("error reading list end: ", err) + } + return nil +} + +func (p *ColumnMetaData) ReadField14(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI64(ctx); err != nil { + return thrift.PrependError("error reading field 14: ", err) + } else { + p.BloomFilterOffset = &v + } + return nil } func (p *ColumnMetaData) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "ColumnMetaData"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - if err := p.writeField3(ctx, oprot); err != nil { return err } - if err := p.writeField4(ctx, oprot); err != nil { return err } - if err := p.writeField5(ctx, oprot); err != nil { return err } - if err := p.writeField6(ctx, oprot); err != nil { return err } - if err := p.writeField7(ctx, oprot); err != nil { return err } - if err := p.writeField8(ctx, oprot); err != nil { return err } - if err := p.writeField9(ctx, oprot); err != nil { return err } - if err := p.writeField10(ctx, oprot); err != nil { return err } - if err := p.writeField11(ctx, oprot); err != nil { return err } - if err := p.writeField12(ctx, oprot); err != nil { return err } - if err := p.writeField13(ctx, oprot); err != nil { return err } - if err := p.writeField14(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "ColumnMetaData"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + if err := p.writeField3(ctx, oprot); err != nil { + return err + } + if err := p.writeField4(ctx, oprot); err != nil { + return err + } + if err := p.writeField5(ctx, oprot); err != nil { + return err + } + if err := p.writeField6(ctx, oprot); err != nil { + return err + } + if err := p.writeField7(ctx, oprot); err != nil { + return err + } + if err := p.writeField8(ctx, oprot); err != nil { + return err + } + if err := p.writeField9(ctx, oprot); err != nil { + return err + } + if err := p.writeField10(ctx, oprot); err != nil { + return err + } + if err := p.writeField11(ctx, oprot); err != nil { + return err + } + if err := p.writeField12(ctx, oprot); err != nil { + return err + } + if err := p.writeField13(ctx, oprot); err != nil { + return err + } + if err := p.writeField14(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *ColumnMetaData) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "type", thrift.I32, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:type: ", p), err) } - if err := oprot.WriteI32(ctx, int32(p.Type)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.type (1) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:type: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "type", thrift.I32, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:type: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.Type)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.type (1) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:type: ", p), err) + } + return err } func (p *ColumnMetaData) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "encodings", thrift.LIST, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:encodings: ", p), err) } - if err := oprot.WriteListBegin(ctx, thrift.I32, len(p.Encodings)); err != nil { - return thrift.PrependError("error writing list begin: ", err) - } - for _, v := range p.Encodings { - if err := oprot.WriteI32(ctx, int32(v)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T. (0) field write error: ", p), err) } - } - if err := oprot.WriteListEnd(ctx); err != nil { - return thrift.PrependError("error writing list end: ", err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:encodings: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "encodings", thrift.LIST, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:encodings: ", p), err) + } + if err := oprot.WriteListBegin(ctx, thrift.I32, len(p.Encodings)); err != nil { + return thrift.PrependError("error writing list begin: ", err) + } + for _, v := range p.Encodings { + if err := oprot.WriteI32(ctx, int32(v)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T. (0) field write error: ", p), err) + } + } + if err := oprot.WriteListEnd(ctx); err != nil { + return thrift.PrependError("error writing list end: ", err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:encodings: ", p), err) + } + return err } func (p *ColumnMetaData) writeField3(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "path_in_schema", thrift.LIST, 3); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:path_in_schema: ", p), err) } - if err := oprot.WriteListBegin(ctx, thrift.STRING, len(p.PathInSchema)); err != nil { - return thrift.PrependError("error writing list begin: ", err) - } - for _, v := range p.PathInSchema { - if err := oprot.WriteString(ctx, string(v)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T. (0) field write error: ", p), err) } - } - if err := oprot.WriteListEnd(ctx); err != nil { - return thrift.PrependError("error writing list end: ", err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 3:path_in_schema: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "path_in_schema", thrift.LIST, 3); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:path_in_schema: ", p), err) + } + if err := oprot.WriteListBegin(ctx, thrift.STRING, len(p.PathInSchema)); err != nil { + return thrift.PrependError("error writing list begin: ", err) + } + for _, v := range p.PathInSchema { + if err := oprot.WriteString(ctx, string(v)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T. (0) field write error: ", p), err) + } + } + if err := oprot.WriteListEnd(ctx); err != nil { + return thrift.PrependError("error writing list end: ", err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 3:path_in_schema: ", p), err) + } + return err } func (p *ColumnMetaData) writeField4(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "codec", thrift.I32, 4); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:codec: ", p), err) } - if err := oprot.WriteI32(ctx, int32(p.Codec)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.codec (4) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 4:codec: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "codec", thrift.I32, 4); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:codec: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.Codec)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.codec (4) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 4:codec: ", p), err) + } + return err } func (p *ColumnMetaData) writeField5(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "num_values", thrift.I64, 5); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:num_values: ", p), err) } - if err := oprot.WriteI64(ctx, int64(p.NumValues)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.num_values (5) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 5:num_values: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "num_values", thrift.I64, 5); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:num_values: ", p), err) + } + if err := oprot.WriteI64(ctx, int64(p.NumValues)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.num_values (5) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 5:num_values: ", p), err) + } + return err } func (p *ColumnMetaData) writeField6(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "total_uncompressed_size", thrift.I64, 6); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 6:total_uncompressed_size: ", p), err) } - if err := oprot.WriteI64(ctx, int64(p.TotalUncompressedSize)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.total_uncompressed_size (6) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 6:total_uncompressed_size: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "total_uncompressed_size", thrift.I64, 6); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 6:total_uncompressed_size: ", p), err) + } + if err := oprot.WriteI64(ctx, int64(p.TotalUncompressedSize)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.total_uncompressed_size (6) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 6:total_uncompressed_size: ", p), err) + } + return err } func (p *ColumnMetaData) writeField7(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "total_compressed_size", thrift.I64, 7); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 7:total_compressed_size: ", p), err) } - if err := oprot.WriteI64(ctx, int64(p.TotalCompressedSize)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.total_compressed_size (7) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 7:total_compressed_size: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "total_compressed_size", thrift.I64, 7); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 7:total_compressed_size: ", p), err) + } + if err := oprot.WriteI64(ctx, int64(p.TotalCompressedSize)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.total_compressed_size (7) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 7:total_compressed_size: ", p), err) + } + return err } func (p *ColumnMetaData) writeField8(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetKeyValueMetadata() { - if err := oprot.WriteFieldBegin(ctx, "key_value_metadata", thrift.LIST, 8); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 8:key_value_metadata: ", p), err) } - if err := oprot.WriteListBegin(ctx, thrift.STRUCT, len(p.KeyValueMetadata)); err != nil { - return thrift.PrependError("error writing list begin: ", err) - } - for _, v := range p.KeyValueMetadata { - if err := v.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", v), err) - } - } - if err := oprot.WriteListEnd(ctx); err != nil { - return thrift.PrependError("error writing list end: ", err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 8:key_value_metadata: ", p), err) } - } - return err + if p.IsSetKeyValueMetadata() { + if err := oprot.WriteFieldBegin(ctx, "key_value_metadata", thrift.LIST, 8); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 8:key_value_metadata: ", p), err) + } + if err := oprot.WriteListBegin(ctx, thrift.STRUCT, len(p.KeyValueMetadata)); err != nil { + return thrift.PrependError("error writing list begin: ", err) + } + for _, v := range p.KeyValueMetadata { + if err := v.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", v), err) + } + } + if err := oprot.WriteListEnd(ctx); err != nil { + return thrift.PrependError("error writing list end: ", err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 8:key_value_metadata: ", p), err) + } + } + return err } func (p *ColumnMetaData) writeField9(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "data_page_offset", thrift.I64, 9); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 9:data_page_offset: ", p), err) } - if err := oprot.WriteI64(ctx, int64(p.DataPageOffset)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.data_page_offset (9) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 9:data_page_offset: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "data_page_offset", thrift.I64, 9); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 9:data_page_offset: ", p), err) + } + if err := oprot.WriteI64(ctx, int64(p.DataPageOffset)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.data_page_offset (9) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 9:data_page_offset: ", p), err) + } + return err } func (p *ColumnMetaData) writeField10(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetIndexPageOffset() { - if err := oprot.WriteFieldBegin(ctx, "index_page_offset", thrift.I64, 10); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 10:index_page_offset: ", p), err) } - if err := oprot.WriteI64(ctx, int64(*p.IndexPageOffset)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.index_page_offset (10) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 10:index_page_offset: ", p), err) } - } - return err + if p.IsSetIndexPageOffset() { + if err := oprot.WriteFieldBegin(ctx, "index_page_offset", thrift.I64, 10); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 10:index_page_offset: ", p), err) + } + if err := oprot.WriteI64(ctx, int64(*p.IndexPageOffset)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.index_page_offset (10) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 10:index_page_offset: ", p), err) + } + } + return err } func (p *ColumnMetaData) writeField11(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetDictionaryPageOffset() { - if err := oprot.WriteFieldBegin(ctx, "dictionary_page_offset", thrift.I64, 11); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 11:dictionary_page_offset: ", p), err) } - if err := oprot.WriteI64(ctx, int64(*p.DictionaryPageOffset)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.dictionary_page_offset (11) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 11:dictionary_page_offset: ", p), err) } - } - return err + if p.IsSetDictionaryPageOffset() { + if err := oprot.WriteFieldBegin(ctx, "dictionary_page_offset", thrift.I64, 11); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 11:dictionary_page_offset: ", p), err) + } + if err := oprot.WriteI64(ctx, int64(*p.DictionaryPageOffset)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.dictionary_page_offset (11) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 11:dictionary_page_offset: ", p), err) + } + } + return err } func (p *ColumnMetaData) writeField12(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetStatistics() { - if err := oprot.WriteFieldBegin(ctx, "statistics", thrift.STRUCT, 12); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 12:statistics: ", p), err) } - if err := p.Statistics.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.Statistics), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 12:statistics: ", p), err) } - } - return err + if p.IsSetStatistics() { + if err := oprot.WriteFieldBegin(ctx, "statistics", thrift.STRUCT, 12); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 12:statistics: ", p), err) + } + if err := p.Statistics.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.Statistics), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 12:statistics: ", p), err) + } + } + return err } func (p *ColumnMetaData) writeField13(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetEncodingStats() { - if err := oprot.WriteFieldBegin(ctx, "encoding_stats", thrift.LIST, 13); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 13:encoding_stats: ", p), err) } - if err := oprot.WriteListBegin(ctx, thrift.STRUCT, len(p.EncodingStats)); err != nil { - return thrift.PrependError("error writing list begin: ", err) - } - for _, v := range p.EncodingStats { - if err := v.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", v), err) - } - } - if err := oprot.WriteListEnd(ctx); err != nil { - return thrift.PrependError("error writing list end: ", err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 13:encoding_stats: ", p), err) } - } - return err + if p.IsSetEncodingStats() { + if err := oprot.WriteFieldBegin(ctx, "encoding_stats", thrift.LIST, 13); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 13:encoding_stats: ", p), err) + } + if err := oprot.WriteListBegin(ctx, thrift.STRUCT, len(p.EncodingStats)); err != nil { + return thrift.PrependError("error writing list begin: ", err) + } + for _, v := range p.EncodingStats { + if err := v.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", v), err) + } + } + if err := oprot.WriteListEnd(ctx); err != nil { + return thrift.PrependError("error writing list end: ", err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 13:encoding_stats: ", p), err) + } + } + return err } func (p *ColumnMetaData) writeField14(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetBloomFilterOffset() { - if err := oprot.WriteFieldBegin(ctx, "bloom_filter_offset", thrift.I64, 14); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 14:bloom_filter_offset: ", p), err) } - if err := oprot.WriteI64(ctx, int64(*p.BloomFilterOffset)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.bloom_filter_offset (14) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 14:bloom_filter_offset: ", p), err) } - } - return err + if p.IsSetBloomFilterOffset() { + if err := oprot.WriteFieldBegin(ctx, "bloom_filter_offset", thrift.I64, 14); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 14:bloom_filter_offset: ", p), err) + } + if err := oprot.WriteI64(ctx, int64(*p.BloomFilterOffset)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.bloom_filter_offset (14) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 14:bloom_filter_offset: ", p), err) + } + } + return err } func (p *ColumnMetaData) Equals(other *ColumnMetaData) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if p.Type != other.Type { return false } - if len(p.Encodings) != len(other.Encodings) { return false } - for i, _tgt := range p.Encodings { - _src4 := other.Encodings[i] - if _tgt != _src4 { return false } - } - if len(p.PathInSchema) != len(other.PathInSchema) { return false } - for i, _tgt := range p.PathInSchema { - _src5 := other.PathInSchema[i] - if _tgt != _src5 { return false } - } - if p.Codec != other.Codec { return false } - if p.NumValues != other.NumValues { return false } - if p.TotalUncompressedSize != other.TotalUncompressedSize { return false } - if p.TotalCompressedSize != other.TotalCompressedSize { return false } - if len(p.KeyValueMetadata) != len(other.KeyValueMetadata) { return false } - for i, _tgt := range p.KeyValueMetadata { - _src6 := other.KeyValueMetadata[i] - if !_tgt.Equals(_src6) { return false } - } - if p.DataPageOffset != other.DataPageOffset { return false } - if p.IndexPageOffset != other.IndexPageOffset { - if p.IndexPageOffset == nil || other.IndexPageOffset == nil { - return false - } - if (*p.IndexPageOffset) != (*other.IndexPageOffset) { return false } - } - if p.DictionaryPageOffset != other.DictionaryPageOffset { - if p.DictionaryPageOffset == nil || other.DictionaryPageOffset == nil { - return false - } - if (*p.DictionaryPageOffset) != (*other.DictionaryPageOffset) { return false } - } - if !p.Statistics.Equals(other.Statistics) { return false } - if len(p.EncodingStats) != len(other.EncodingStats) { return false } - for i, _tgt := range p.EncodingStats { - _src7 := other.EncodingStats[i] - if !_tgt.Equals(_src7) { return false } - } - if p.BloomFilterOffset != other.BloomFilterOffset { - if p.BloomFilterOffset == nil || other.BloomFilterOffset == nil { - return false - } - if (*p.BloomFilterOffset) != (*other.BloomFilterOffset) { return false } - } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if p.Type != other.Type { + return false + } + if len(p.Encodings) != len(other.Encodings) { + return false + } + for i, _tgt := range p.Encodings { + _src4 := other.Encodings[i] + if _tgt != _src4 { + return false + } + } + if len(p.PathInSchema) != len(other.PathInSchema) { + return false + } + for i, _tgt := range p.PathInSchema { + _src5 := other.PathInSchema[i] + if _tgt != _src5 { + return false + } + } + if p.Codec != other.Codec { + return false + } + if p.NumValues != other.NumValues { + return false + } + if p.TotalUncompressedSize != other.TotalUncompressedSize { + return false + } + if p.TotalCompressedSize != other.TotalCompressedSize { + return false + } + if len(p.KeyValueMetadata) != len(other.KeyValueMetadata) { + return false + } + for i, _tgt := range p.KeyValueMetadata { + _src6 := other.KeyValueMetadata[i] + if !_tgt.Equals(_src6) { + return false + } + } + if p.DataPageOffset != other.DataPageOffset { + return false + } + if p.IndexPageOffset != other.IndexPageOffset { + if p.IndexPageOffset == nil || other.IndexPageOffset == nil { + return false + } + if (*p.IndexPageOffset) != (*other.IndexPageOffset) { + return false + } + } + if p.DictionaryPageOffset != other.DictionaryPageOffset { + if p.DictionaryPageOffset == nil || other.DictionaryPageOffset == nil { + return false + } + if (*p.DictionaryPageOffset) != (*other.DictionaryPageOffset) { + return false + } + } + if !p.Statistics.Equals(other.Statistics) { + return false + } + if len(p.EncodingStats) != len(other.EncodingStats) { + return false + } + for i, _tgt := range p.EncodingStats { + _src7 := other.EncodingStats[i] + if !_tgt.Equals(_src7) { + return false + } + } + if p.BloomFilterOffset != other.BloomFilterOffset { + if p.BloomFilterOffset == nil || other.BloomFilterOffset == nil { + return false + } + if (*p.BloomFilterOffset) != (*other.BloomFilterOffset) { + return false + } + } + return true } func (p *ColumnMetaData) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("ColumnMetaData(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("ColumnMetaData(%+v)", *p) } func (p *ColumnMetaData) Validate() error { - return nil + return nil } + type EncryptionWithFooterKey struct { } func NewEncryptionWithFooterKey() *EncryptionWithFooterKey { - return &EncryptionWithFooterKey{} + return &EncryptionWithFooterKey{} } func (p *EncryptionWithFooterKey) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil } func (p *EncryptionWithFooterKey) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "EncryptionWithFooterKey"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "EncryptionWithFooterKey"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *EncryptionWithFooterKey) Equals(other *EncryptionWithFooterKey) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + return true } func (p *EncryptionWithFooterKey) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("EncryptionWithFooterKey(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("EncryptionWithFooterKey(%+v)", *p) } func (p *EncryptionWithFooterKey) Validate() error { - return nil + return nil } + // Attributes: -// - PathInSchema: Column path in schema * -// - KeyMetadata: Retrieval metadata of column encryption key * +// - PathInSchema: Column path in schema * +// - KeyMetadata: Retrieval metadata of column encryption key * type EncryptionWithColumnKey struct { - PathInSchema []string `thrift:"path_in_schema,1,required" db:"path_in_schema" json:"path_in_schema"` - KeyMetadata []byte `thrift:"key_metadata,2" db:"key_metadata" json:"key_metadata,omitempty"` + PathInSchema []string `thrift:"path_in_schema,1,required" db:"path_in_schema" json:"path_in_schema"` + KeyMetadata []byte `thrift:"key_metadata,2" db:"key_metadata" json:"key_metadata,omitempty"` } func NewEncryptionWithColumnKey() *EncryptionWithColumnKey { - return &EncryptionWithColumnKey{} + return &EncryptionWithColumnKey{} } - func (p *EncryptionWithColumnKey) GetPathInSchema() []string { - return p.PathInSchema + return p.PathInSchema } + var EncryptionWithColumnKey_KeyMetadata_DEFAULT []byte func (p *EncryptionWithColumnKey) GetKeyMetadata() []byte { - return p.KeyMetadata + return p.KeyMetadata } func (p *EncryptionWithColumnKey) IsSetKeyMetadata() bool { - return p.KeyMetadata != nil + return p.KeyMetadata != nil } func (p *EncryptionWithColumnKey) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - var issetPathInSchema bool = false; - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.LIST { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - issetPathInSchema = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.STRING { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - if !issetPathInSchema{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field PathInSchema is not set")); - } - return nil -} - -func (p *EncryptionWithColumnKey) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - _, size, err := iprot.ReadListBegin(ctx) - if err != nil { - return thrift.PrependError("error reading list begin: ", err) - } - tSlice := make([]string, 0, size) - p.PathInSchema = tSlice - for i := 0; i < size; i ++ { -var _elem8 string - if v, err := iprot.ReadString(ctx); err != nil { - return thrift.PrependError("error reading field 0: ", err) -} else { - _elem8 = v -} - p.PathInSchema = append(p.PathInSchema, _elem8) - } - if err := iprot.ReadListEnd(ctx); err != nil { - return thrift.PrependError("error reading list end: ", err) - } - return nil -} - -func (p *EncryptionWithColumnKey) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadBinary(ctx); err != nil { - return thrift.PrependError("error reading field 2: ", err) -} else { - p.KeyMetadata = v -} - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + var issetPathInSchema bool = false + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.LIST { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + issetPathInSchema = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.STRING { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + if !issetPathInSchema { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field PathInSchema is not set")) + } + return nil +} + +func (p *EncryptionWithColumnKey) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + _, size, err := iprot.ReadListBegin(ctx) + if err != nil { + return thrift.PrependError("error reading list begin: ", err) + } + tSlice := make([]string, 0, size) + p.PathInSchema = tSlice + for i := 0; i < size; i++ { + var _elem8 string + if v, err := iprot.ReadString(ctx); err != nil { + return thrift.PrependError("error reading field 0: ", err) + } else { + _elem8 = v + } + p.PathInSchema = append(p.PathInSchema, _elem8) + } + if err := iprot.ReadListEnd(ctx); err != nil { + return thrift.PrependError("error reading list end: ", err) + } + return nil +} + +func (p *EncryptionWithColumnKey) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadBinary(ctx); err != nil { + return thrift.PrependError("error reading field 2: ", err) + } else { + p.KeyMetadata = v + } + return nil } func (p *EncryptionWithColumnKey) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "EncryptionWithColumnKey"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "EncryptionWithColumnKey"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *EncryptionWithColumnKey) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "path_in_schema", thrift.LIST, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:path_in_schema: ", p), err) } - if err := oprot.WriteListBegin(ctx, thrift.STRING, len(p.PathInSchema)); err != nil { - return thrift.PrependError("error writing list begin: ", err) - } - for _, v := range p.PathInSchema { - if err := oprot.WriteString(ctx, string(v)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T. (0) field write error: ", p), err) } - } - if err := oprot.WriteListEnd(ctx); err != nil { - return thrift.PrependError("error writing list end: ", err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:path_in_schema: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "path_in_schema", thrift.LIST, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:path_in_schema: ", p), err) + } + if err := oprot.WriteListBegin(ctx, thrift.STRING, len(p.PathInSchema)); err != nil { + return thrift.PrependError("error writing list begin: ", err) + } + for _, v := range p.PathInSchema { + if err := oprot.WriteString(ctx, string(v)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T. (0) field write error: ", p), err) + } + } + if err := oprot.WriteListEnd(ctx); err != nil { + return thrift.PrependError("error writing list end: ", err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:path_in_schema: ", p), err) + } + return err } func (p *EncryptionWithColumnKey) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetKeyMetadata() { - if err := oprot.WriteFieldBegin(ctx, "key_metadata", thrift.STRING, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:key_metadata: ", p), err) } - if err := oprot.WriteBinary(ctx, p.KeyMetadata); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.key_metadata (2) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:key_metadata: ", p), err) } - } - return err + if p.IsSetKeyMetadata() { + if err := oprot.WriteFieldBegin(ctx, "key_metadata", thrift.STRING, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:key_metadata: ", p), err) + } + if err := oprot.WriteBinary(ctx, p.KeyMetadata); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.key_metadata (2) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:key_metadata: ", p), err) + } + } + return err } func (p *EncryptionWithColumnKey) Equals(other *EncryptionWithColumnKey) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if len(p.PathInSchema) != len(other.PathInSchema) { return false } - for i, _tgt := range p.PathInSchema { - _src9 := other.PathInSchema[i] - if _tgt != _src9 { return false } - } - if bytes.Compare(p.KeyMetadata, other.KeyMetadata) != 0 { return false } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if len(p.PathInSchema) != len(other.PathInSchema) { + return false + } + for i, _tgt := range p.PathInSchema { + _src9 := other.PathInSchema[i] + if _tgt != _src9 { + return false + } + } + if bytes.Compare(p.KeyMetadata, other.KeyMetadata) != 0 { + return false + } + return true } func (p *EncryptionWithColumnKey) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("EncryptionWithColumnKey(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("EncryptionWithColumnKey(%+v)", *p) } func (p *EncryptionWithColumnKey) Validate() error { - return nil + return nil } + // Attributes: -// - ENCRYPTION_WITH_FOOTER_KEY -// - ENCRYPTION_WITH_COLUMN_KEY +// - ENCRYPTION_WITH_FOOTER_KEY +// - ENCRYPTION_WITH_COLUMN_KEY type ColumnCryptoMetaData struct { - ENCRYPTION_WITH_FOOTER_KEY *EncryptionWithFooterKey `thrift:"ENCRYPTION_WITH_FOOTER_KEY,1" db:"ENCRYPTION_WITH_FOOTER_KEY" json:"ENCRYPTION_WITH_FOOTER_KEY,omitempty"` - ENCRYPTION_WITH_COLUMN_KEY *EncryptionWithColumnKey `thrift:"ENCRYPTION_WITH_COLUMN_KEY,2" db:"ENCRYPTION_WITH_COLUMN_KEY" json:"ENCRYPTION_WITH_COLUMN_KEY,omitempty"` + ENCRYPTION_WITH_FOOTER_KEY *EncryptionWithFooterKey `thrift:"ENCRYPTION_WITH_FOOTER_KEY,1" db:"ENCRYPTION_WITH_FOOTER_KEY" json:"ENCRYPTION_WITH_FOOTER_KEY,omitempty"` + ENCRYPTION_WITH_COLUMN_KEY *EncryptionWithColumnKey `thrift:"ENCRYPTION_WITH_COLUMN_KEY,2" db:"ENCRYPTION_WITH_COLUMN_KEY" json:"ENCRYPTION_WITH_COLUMN_KEY,omitempty"` } func NewColumnCryptoMetaData() *ColumnCryptoMetaData { - return &ColumnCryptoMetaData{} + return &ColumnCryptoMetaData{} } var ColumnCryptoMetaData_ENCRYPTION_WITH_FOOTER_KEY_DEFAULT *EncryptionWithFooterKey + func (p *ColumnCryptoMetaData) GetENCRYPTION_WITH_FOOTER_KEY() *EncryptionWithFooterKey { - if !p.IsSetENCRYPTION_WITH_FOOTER_KEY() { - return ColumnCryptoMetaData_ENCRYPTION_WITH_FOOTER_KEY_DEFAULT - } -return p.ENCRYPTION_WITH_FOOTER_KEY + if !p.IsSetENCRYPTION_WITH_FOOTER_KEY() { + return ColumnCryptoMetaData_ENCRYPTION_WITH_FOOTER_KEY_DEFAULT + } + return p.ENCRYPTION_WITH_FOOTER_KEY } + var ColumnCryptoMetaData_ENCRYPTION_WITH_COLUMN_KEY_DEFAULT *EncryptionWithColumnKey + func (p *ColumnCryptoMetaData) GetENCRYPTION_WITH_COLUMN_KEY() *EncryptionWithColumnKey { - if !p.IsSetENCRYPTION_WITH_COLUMN_KEY() { - return ColumnCryptoMetaData_ENCRYPTION_WITH_COLUMN_KEY_DEFAULT - } -return p.ENCRYPTION_WITH_COLUMN_KEY + if !p.IsSetENCRYPTION_WITH_COLUMN_KEY() { + return ColumnCryptoMetaData_ENCRYPTION_WITH_COLUMN_KEY_DEFAULT + } + return p.ENCRYPTION_WITH_COLUMN_KEY } func (p *ColumnCryptoMetaData) CountSetFieldsColumnCryptoMetaData() int { - count := 0 - if (p.IsSetENCRYPTION_WITH_FOOTER_KEY()) { - count++ - } - if (p.IsSetENCRYPTION_WITH_COLUMN_KEY()) { - count++ - } - return count + count := 0 + if p.IsSetENCRYPTION_WITH_FOOTER_KEY() { + count++ + } + if p.IsSetENCRYPTION_WITH_COLUMN_KEY() { + count++ + } + return count } func (p *ColumnCryptoMetaData) IsSetENCRYPTION_WITH_FOOTER_KEY() bool { - return p.ENCRYPTION_WITH_FOOTER_KEY != nil + return p.ENCRYPTION_WITH_FOOTER_KEY != nil } func (p *ColumnCryptoMetaData) IsSetENCRYPTION_WITH_COLUMN_KEY() bool { - return p.ENCRYPTION_WITH_COLUMN_KEY != nil + return p.ENCRYPTION_WITH_COLUMN_KEY != nil } func (p *ColumnCryptoMetaData) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil -} - -func (p *ColumnCryptoMetaData) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - p.ENCRYPTION_WITH_FOOTER_KEY = &EncryptionWithFooterKey{} - if err := p.ENCRYPTION_WITH_FOOTER_KEY.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.ENCRYPTION_WITH_FOOTER_KEY), err) - } - return nil -} - -func (p *ColumnCryptoMetaData) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - p.ENCRYPTION_WITH_COLUMN_KEY = &EncryptionWithColumnKey{} - if err := p.ENCRYPTION_WITH_COLUMN_KEY.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.ENCRYPTION_WITH_COLUMN_KEY), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil +} + +func (p *ColumnCryptoMetaData) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + p.ENCRYPTION_WITH_FOOTER_KEY = &EncryptionWithFooterKey{} + if err := p.ENCRYPTION_WITH_FOOTER_KEY.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.ENCRYPTION_WITH_FOOTER_KEY), err) + } + return nil +} + +func (p *ColumnCryptoMetaData) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + p.ENCRYPTION_WITH_COLUMN_KEY = &EncryptionWithColumnKey{} + if err := p.ENCRYPTION_WITH_COLUMN_KEY.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.ENCRYPTION_WITH_COLUMN_KEY), err) + } + return nil } func (p *ColumnCryptoMetaData) Write(ctx context.Context, oprot thrift.TProtocol) error { - if c := p.CountSetFieldsColumnCryptoMetaData(); c != 1 { - return fmt.Errorf("%T write union: exactly one field must be set (%d set)", p, c) - } - if err := oprot.WriteStructBegin(ctx, "ColumnCryptoMetaData"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if c := p.CountSetFieldsColumnCryptoMetaData(); c != 1 { + return fmt.Errorf("%T write union: exactly one field must be set (%d set)", p, c) + } + if err := oprot.WriteStructBegin(ctx, "ColumnCryptoMetaData"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *ColumnCryptoMetaData) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetENCRYPTION_WITH_FOOTER_KEY() { - if err := oprot.WriteFieldBegin(ctx, "ENCRYPTION_WITH_FOOTER_KEY", thrift.STRUCT, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:ENCRYPTION_WITH_FOOTER_KEY: ", p), err) } - if err := p.ENCRYPTION_WITH_FOOTER_KEY.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.ENCRYPTION_WITH_FOOTER_KEY), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:ENCRYPTION_WITH_FOOTER_KEY: ", p), err) } - } - return err + if p.IsSetENCRYPTION_WITH_FOOTER_KEY() { + if err := oprot.WriteFieldBegin(ctx, "ENCRYPTION_WITH_FOOTER_KEY", thrift.STRUCT, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:ENCRYPTION_WITH_FOOTER_KEY: ", p), err) + } + if err := p.ENCRYPTION_WITH_FOOTER_KEY.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.ENCRYPTION_WITH_FOOTER_KEY), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:ENCRYPTION_WITH_FOOTER_KEY: ", p), err) + } + } + return err } func (p *ColumnCryptoMetaData) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetENCRYPTION_WITH_COLUMN_KEY() { - if err := oprot.WriteFieldBegin(ctx, "ENCRYPTION_WITH_COLUMN_KEY", thrift.STRUCT, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:ENCRYPTION_WITH_COLUMN_KEY: ", p), err) } - if err := p.ENCRYPTION_WITH_COLUMN_KEY.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.ENCRYPTION_WITH_COLUMN_KEY), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:ENCRYPTION_WITH_COLUMN_KEY: ", p), err) } - } - return err + if p.IsSetENCRYPTION_WITH_COLUMN_KEY() { + if err := oprot.WriteFieldBegin(ctx, "ENCRYPTION_WITH_COLUMN_KEY", thrift.STRUCT, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:ENCRYPTION_WITH_COLUMN_KEY: ", p), err) + } + if err := p.ENCRYPTION_WITH_COLUMN_KEY.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.ENCRYPTION_WITH_COLUMN_KEY), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:ENCRYPTION_WITH_COLUMN_KEY: ", p), err) + } + } + return err } func (p *ColumnCryptoMetaData) Equals(other *ColumnCryptoMetaData) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if !p.ENCRYPTION_WITH_FOOTER_KEY.Equals(other.ENCRYPTION_WITH_FOOTER_KEY) { return false } - if !p.ENCRYPTION_WITH_COLUMN_KEY.Equals(other.ENCRYPTION_WITH_COLUMN_KEY) { return false } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if !p.ENCRYPTION_WITH_FOOTER_KEY.Equals(other.ENCRYPTION_WITH_FOOTER_KEY) { + return false + } + if !p.ENCRYPTION_WITH_COLUMN_KEY.Equals(other.ENCRYPTION_WITH_COLUMN_KEY) { + return false + } + return true } func (p *ColumnCryptoMetaData) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("ColumnCryptoMetaData(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("ColumnCryptoMetaData(%+v)", *p) } func (p *ColumnCryptoMetaData) Validate() error { - return nil + return nil } + // Attributes: -// - FilePath: File where column data is stored. If not set, assumed to be same file as +// - FilePath: File where column data is stored. If not set, assumed to be same file as +// // metadata. This path is relative to the current file. -// -// - FileOffset: Byte offset in file_path to the ColumnMetaData * -// - MetaData: Column metadata for this chunk. This is the same content as what is at +// +// - FileOffset: Byte offset in file_path to the ColumnMetaData * +// - MetaData: Column metadata for this chunk. This is the same content as what is at +// // file_path/file_offset. Having it here has it replicated in the file // metadata. -// -// - OffsetIndexOffset: File offset of ColumnChunk's OffsetIndex * -// - OffsetIndexLength: Size of ColumnChunk's OffsetIndex, in bytes * -// - ColumnIndexOffset: File offset of ColumnChunk's ColumnIndex * -// - ColumnIndexLength: Size of ColumnChunk's ColumnIndex, in bytes * -// - CryptoMetadata: Crypto metadata of encrypted columns * -// - EncryptedColumnMetadata: Encrypted column metadata for this chunk * +// +// - OffsetIndexOffset: File offset of ColumnChunk's OffsetIndex * +// - OffsetIndexLength: Size of ColumnChunk's OffsetIndex, in bytes * +// - ColumnIndexOffset: File offset of ColumnChunk's ColumnIndex * +// - ColumnIndexLength: Size of ColumnChunk's ColumnIndex, in bytes * +// - CryptoMetadata: Crypto metadata of encrypted columns * +// - EncryptedColumnMetadata: Encrypted column metadata for this chunk * type ColumnChunk struct { - FilePath *string `thrift:"file_path,1" db:"file_path" json:"file_path,omitempty"` - FileOffset int64 `thrift:"file_offset,2,required" db:"file_offset" json:"file_offset"` - MetaData *ColumnMetaData `thrift:"meta_data,3" db:"meta_data" json:"meta_data,omitempty"` - OffsetIndexOffset *int64 `thrift:"offset_index_offset,4" db:"offset_index_offset" json:"offset_index_offset,omitempty"` - OffsetIndexLength *int32 `thrift:"offset_index_length,5" db:"offset_index_length" json:"offset_index_length,omitempty"` - ColumnIndexOffset *int64 `thrift:"column_index_offset,6" db:"column_index_offset" json:"column_index_offset,omitempty"` - ColumnIndexLength *int32 `thrift:"column_index_length,7" db:"column_index_length" json:"column_index_length,omitempty"` - CryptoMetadata *ColumnCryptoMetaData `thrift:"crypto_metadata,8" db:"crypto_metadata" json:"crypto_metadata,omitempty"` - EncryptedColumnMetadata []byte `thrift:"encrypted_column_metadata,9" db:"encrypted_column_metadata" json:"encrypted_column_metadata,omitempty"` + FilePath *string `thrift:"file_path,1" db:"file_path" json:"file_path,omitempty"` + FileOffset int64 `thrift:"file_offset,2,required" db:"file_offset" json:"file_offset"` + MetaData *ColumnMetaData `thrift:"meta_data,3" db:"meta_data" json:"meta_data,omitempty"` + OffsetIndexOffset *int64 `thrift:"offset_index_offset,4" db:"offset_index_offset" json:"offset_index_offset,omitempty"` + OffsetIndexLength *int32 `thrift:"offset_index_length,5" db:"offset_index_length" json:"offset_index_length,omitempty"` + ColumnIndexOffset *int64 `thrift:"column_index_offset,6" db:"column_index_offset" json:"column_index_offset,omitempty"` + ColumnIndexLength *int32 `thrift:"column_index_length,7" db:"column_index_length" json:"column_index_length,omitempty"` + CryptoMetadata *ColumnCryptoMetaData `thrift:"crypto_metadata,8" db:"crypto_metadata" json:"crypto_metadata,omitempty"` + EncryptedColumnMetadata []byte `thrift:"encrypted_column_metadata,9" db:"encrypted_column_metadata" json:"encrypted_column_metadata,omitempty"` } func NewColumnChunk() *ColumnChunk { - return &ColumnChunk{} + return &ColumnChunk{} } var ColumnChunk_FilePath_DEFAULT string + func (p *ColumnChunk) GetFilePath() string { - if !p.IsSetFilePath() { - return ColumnChunk_FilePath_DEFAULT - } -return *p.FilePath + if !p.IsSetFilePath() { + return ColumnChunk_FilePath_DEFAULT + } + return *p.FilePath } func (p *ColumnChunk) GetFileOffset() int64 { - return p.FileOffset + return p.FileOffset } + var ColumnChunk_MetaData_DEFAULT *ColumnMetaData + func (p *ColumnChunk) GetMetaData() *ColumnMetaData { - if !p.IsSetMetaData() { - return ColumnChunk_MetaData_DEFAULT - } -return p.MetaData + if !p.IsSetMetaData() { + return ColumnChunk_MetaData_DEFAULT + } + return p.MetaData } + var ColumnChunk_OffsetIndexOffset_DEFAULT int64 + func (p *ColumnChunk) GetOffsetIndexOffset() int64 { - if !p.IsSetOffsetIndexOffset() { - return ColumnChunk_OffsetIndexOffset_DEFAULT - } -return *p.OffsetIndexOffset + if !p.IsSetOffsetIndexOffset() { + return ColumnChunk_OffsetIndexOffset_DEFAULT + } + return *p.OffsetIndexOffset } + var ColumnChunk_OffsetIndexLength_DEFAULT int32 + func (p *ColumnChunk) GetOffsetIndexLength() int32 { - if !p.IsSetOffsetIndexLength() { - return ColumnChunk_OffsetIndexLength_DEFAULT - } -return *p.OffsetIndexLength + if !p.IsSetOffsetIndexLength() { + return ColumnChunk_OffsetIndexLength_DEFAULT + } + return *p.OffsetIndexLength } + var ColumnChunk_ColumnIndexOffset_DEFAULT int64 + func (p *ColumnChunk) GetColumnIndexOffset() int64 { - if !p.IsSetColumnIndexOffset() { - return ColumnChunk_ColumnIndexOffset_DEFAULT - } -return *p.ColumnIndexOffset + if !p.IsSetColumnIndexOffset() { + return ColumnChunk_ColumnIndexOffset_DEFAULT + } + return *p.ColumnIndexOffset } + var ColumnChunk_ColumnIndexLength_DEFAULT int32 + func (p *ColumnChunk) GetColumnIndexLength() int32 { - if !p.IsSetColumnIndexLength() { - return ColumnChunk_ColumnIndexLength_DEFAULT - } -return *p.ColumnIndexLength + if !p.IsSetColumnIndexLength() { + return ColumnChunk_ColumnIndexLength_DEFAULT + } + return *p.ColumnIndexLength } + var ColumnChunk_CryptoMetadata_DEFAULT *ColumnCryptoMetaData + func (p *ColumnChunk) GetCryptoMetadata() *ColumnCryptoMetaData { - if !p.IsSetCryptoMetadata() { - return ColumnChunk_CryptoMetadata_DEFAULT - } -return p.CryptoMetadata + if !p.IsSetCryptoMetadata() { + return ColumnChunk_CryptoMetadata_DEFAULT + } + return p.CryptoMetadata } + var ColumnChunk_EncryptedColumnMetadata_DEFAULT []byte func (p *ColumnChunk) GetEncryptedColumnMetadata() []byte { - return p.EncryptedColumnMetadata + return p.EncryptedColumnMetadata } func (p *ColumnChunk) IsSetFilePath() bool { - return p.FilePath != nil + return p.FilePath != nil } func (p *ColumnChunk) IsSetMetaData() bool { - return p.MetaData != nil + return p.MetaData != nil } func (p *ColumnChunk) IsSetOffsetIndexOffset() bool { - return p.OffsetIndexOffset != nil + return p.OffsetIndexOffset != nil } func (p *ColumnChunk) IsSetOffsetIndexLength() bool { - return p.OffsetIndexLength != nil + return p.OffsetIndexLength != nil } func (p *ColumnChunk) IsSetColumnIndexOffset() bool { - return p.ColumnIndexOffset != nil + return p.ColumnIndexOffset != nil } func (p *ColumnChunk) IsSetColumnIndexLength() bool { - return p.ColumnIndexLength != nil + return p.ColumnIndexLength != nil } func (p *ColumnChunk) IsSetCryptoMetadata() bool { - return p.CryptoMetadata != nil + return p.CryptoMetadata != nil } func (p *ColumnChunk) IsSetEncryptedColumnMetadata() bool { - return p.EncryptedColumnMetadata != nil + return p.EncryptedColumnMetadata != nil } func (p *ColumnChunk) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - var issetFileOffset bool = false; - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.STRING { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.I64 { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - issetFileOffset = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 3: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField3(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 4: - if fieldTypeId == thrift.I64 { - if err := p.ReadField4(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 5: - if fieldTypeId == thrift.I32 { - if err := p.ReadField5(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 6: - if fieldTypeId == thrift.I64 { - if err := p.ReadField6(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 7: - if fieldTypeId == thrift.I32 { - if err := p.ReadField7(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 8: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField8(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 9: - if fieldTypeId == thrift.STRING { - if err := p.ReadField9(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - if !issetFileOffset{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field FileOffset is not set")); - } - return nil -} - -func (p *ColumnChunk) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadString(ctx); err != nil { - return thrift.PrependError("error reading field 1: ", err) -} else { - p.FilePath = &v -} - return nil -} - -func (p *ColumnChunk) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(ctx); err != nil { - return thrift.PrependError("error reading field 2: ", err) -} else { - p.FileOffset = v -} - return nil -} - -func (p *ColumnChunk) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { - p.MetaData = &ColumnMetaData{} - if err := p.MetaData.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.MetaData), err) - } - return nil -} - -func (p *ColumnChunk) ReadField4(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(ctx); err != nil { - return thrift.PrependError("error reading field 4: ", err) -} else { - p.OffsetIndexOffset = &v -} - return nil -} - -func (p *ColumnChunk) ReadField5(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 5: ", err) -} else { - p.OffsetIndexLength = &v -} - return nil -} - -func (p *ColumnChunk) ReadField6(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(ctx); err != nil { - return thrift.PrependError("error reading field 6: ", err) -} else { - p.ColumnIndexOffset = &v -} - return nil -} - -func (p *ColumnChunk) ReadField7(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 7: ", err) -} else { - p.ColumnIndexLength = &v -} - return nil -} - -func (p *ColumnChunk) ReadField8(ctx context.Context, iprot thrift.TProtocol) error { - p.CryptoMetadata = &ColumnCryptoMetaData{} - if err := p.CryptoMetadata.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.CryptoMetadata), err) - } - return nil -} - -func (p *ColumnChunk) ReadField9(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadBinary(ctx); err != nil { - return thrift.PrependError("error reading field 9: ", err) -} else { - p.EncryptedColumnMetadata = v -} - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + var issetFileOffset bool = false + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.STRING { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.I64 { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + issetFileOffset = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 3: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField3(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 4: + if fieldTypeId == thrift.I64 { + if err := p.ReadField4(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 5: + if fieldTypeId == thrift.I32 { + if err := p.ReadField5(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 6: + if fieldTypeId == thrift.I64 { + if err := p.ReadField6(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 7: + if fieldTypeId == thrift.I32 { + if err := p.ReadField7(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 8: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField8(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 9: + if fieldTypeId == thrift.STRING { + if err := p.ReadField9(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + if !issetFileOffset { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field FileOffset is not set")) + } + return nil +} + +func (p *ColumnChunk) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadString(ctx); err != nil { + return thrift.PrependError("error reading field 1: ", err) + } else { + p.FilePath = &v + } + return nil +} + +func (p *ColumnChunk) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI64(ctx); err != nil { + return thrift.PrependError("error reading field 2: ", err) + } else { + p.FileOffset = v + } + return nil +} + +func (p *ColumnChunk) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { + p.MetaData = &ColumnMetaData{} + if err := p.MetaData.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.MetaData), err) + } + return nil +} + +func (p *ColumnChunk) ReadField4(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI64(ctx); err != nil { + return thrift.PrependError("error reading field 4: ", err) + } else { + p.OffsetIndexOffset = &v + } + return nil +} + +func (p *ColumnChunk) ReadField5(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 5: ", err) + } else { + p.OffsetIndexLength = &v + } + return nil +} + +func (p *ColumnChunk) ReadField6(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI64(ctx); err != nil { + return thrift.PrependError("error reading field 6: ", err) + } else { + p.ColumnIndexOffset = &v + } + return nil +} + +func (p *ColumnChunk) ReadField7(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 7: ", err) + } else { + p.ColumnIndexLength = &v + } + return nil +} + +func (p *ColumnChunk) ReadField8(ctx context.Context, iprot thrift.TProtocol) error { + p.CryptoMetadata = &ColumnCryptoMetaData{} + if err := p.CryptoMetadata.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.CryptoMetadata), err) + } + return nil +} + +func (p *ColumnChunk) ReadField9(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadBinary(ctx); err != nil { + return thrift.PrependError("error reading field 9: ", err) + } else { + p.EncryptedColumnMetadata = v + } + return nil } func (p *ColumnChunk) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "ColumnChunk"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - if err := p.writeField3(ctx, oprot); err != nil { return err } - if err := p.writeField4(ctx, oprot); err != nil { return err } - if err := p.writeField5(ctx, oprot); err != nil { return err } - if err := p.writeField6(ctx, oprot); err != nil { return err } - if err := p.writeField7(ctx, oprot); err != nil { return err } - if err := p.writeField8(ctx, oprot); err != nil { return err } - if err := p.writeField9(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "ColumnChunk"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + if err := p.writeField3(ctx, oprot); err != nil { + return err + } + if err := p.writeField4(ctx, oprot); err != nil { + return err + } + if err := p.writeField5(ctx, oprot); err != nil { + return err + } + if err := p.writeField6(ctx, oprot); err != nil { + return err + } + if err := p.writeField7(ctx, oprot); err != nil { + return err + } + if err := p.writeField8(ctx, oprot); err != nil { + return err + } + if err := p.writeField9(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *ColumnChunk) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetFilePath() { - if err := oprot.WriteFieldBegin(ctx, "file_path", thrift.STRING, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:file_path: ", p), err) } - if err := oprot.WriteString(ctx, string(*p.FilePath)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.file_path (1) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:file_path: ", p), err) } - } - return err + if p.IsSetFilePath() { + if err := oprot.WriteFieldBegin(ctx, "file_path", thrift.STRING, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:file_path: ", p), err) + } + if err := oprot.WriteString(ctx, string(*p.FilePath)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.file_path (1) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:file_path: ", p), err) + } + } + return err } func (p *ColumnChunk) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "file_offset", thrift.I64, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:file_offset: ", p), err) } - if err := oprot.WriteI64(ctx, int64(p.FileOffset)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.file_offset (2) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:file_offset: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "file_offset", thrift.I64, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:file_offset: ", p), err) + } + if err := oprot.WriteI64(ctx, int64(p.FileOffset)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.file_offset (2) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:file_offset: ", p), err) + } + return err } func (p *ColumnChunk) writeField3(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetMetaData() { - if err := oprot.WriteFieldBegin(ctx, "meta_data", thrift.STRUCT, 3); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:meta_data: ", p), err) } - if err := p.MetaData.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.MetaData), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 3:meta_data: ", p), err) } - } - return err + if p.IsSetMetaData() { + if err := oprot.WriteFieldBegin(ctx, "meta_data", thrift.STRUCT, 3); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:meta_data: ", p), err) + } + if err := p.MetaData.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.MetaData), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 3:meta_data: ", p), err) + } + } + return err } func (p *ColumnChunk) writeField4(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetOffsetIndexOffset() { - if err := oprot.WriteFieldBegin(ctx, "offset_index_offset", thrift.I64, 4); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:offset_index_offset: ", p), err) } - if err := oprot.WriteI64(ctx, int64(*p.OffsetIndexOffset)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.offset_index_offset (4) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 4:offset_index_offset: ", p), err) } - } - return err + if p.IsSetOffsetIndexOffset() { + if err := oprot.WriteFieldBegin(ctx, "offset_index_offset", thrift.I64, 4); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:offset_index_offset: ", p), err) + } + if err := oprot.WriteI64(ctx, int64(*p.OffsetIndexOffset)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.offset_index_offset (4) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 4:offset_index_offset: ", p), err) + } + } + return err } func (p *ColumnChunk) writeField5(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetOffsetIndexLength() { - if err := oprot.WriteFieldBegin(ctx, "offset_index_length", thrift.I32, 5); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:offset_index_length: ", p), err) } - if err := oprot.WriteI32(ctx, int32(*p.OffsetIndexLength)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.offset_index_length (5) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 5:offset_index_length: ", p), err) } - } - return err + if p.IsSetOffsetIndexLength() { + if err := oprot.WriteFieldBegin(ctx, "offset_index_length", thrift.I32, 5); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:offset_index_length: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(*p.OffsetIndexLength)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.offset_index_length (5) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 5:offset_index_length: ", p), err) + } + } + return err } func (p *ColumnChunk) writeField6(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetColumnIndexOffset() { - if err := oprot.WriteFieldBegin(ctx, "column_index_offset", thrift.I64, 6); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 6:column_index_offset: ", p), err) } - if err := oprot.WriteI64(ctx, int64(*p.ColumnIndexOffset)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.column_index_offset (6) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 6:column_index_offset: ", p), err) } - } - return err + if p.IsSetColumnIndexOffset() { + if err := oprot.WriteFieldBegin(ctx, "column_index_offset", thrift.I64, 6); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 6:column_index_offset: ", p), err) + } + if err := oprot.WriteI64(ctx, int64(*p.ColumnIndexOffset)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.column_index_offset (6) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 6:column_index_offset: ", p), err) + } + } + return err } func (p *ColumnChunk) writeField7(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetColumnIndexLength() { - if err := oprot.WriteFieldBegin(ctx, "column_index_length", thrift.I32, 7); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 7:column_index_length: ", p), err) } - if err := oprot.WriteI32(ctx, int32(*p.ColumnIndexLength)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.column_index_length (7) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 7:column_index_length: ", p), err) } - } - return err + if p.IsSetColumnIndexLength() { + if err := oprot.WriteFieldBegin(ctx, "column_index_length", thrift.I32, 7); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 7:column_index_length: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(*p.ColumnIndexLength)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.column_index_length (7) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 7:column_index_length: ", p), err) + } + } + return err } func (p *ColumnChunk) writeField8(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetCryptoMetadata() { - if err := oprot.WriteFieldBegin(ctx, "crypto_metadata", thrift.STRUCT, 8); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 8:crypto_metadata: ", p), err) } - if err := p.CryptoMetadata.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.CryptoMetadata), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 8:crypto_metadata: ", p), err) } - } - return err + if p.IsSetCryptoMetadata() { + if err := oprot.WriteFieldBegin(ctx, "crypto_metadata", thrift.STRUCT, 8); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 8:crypto_metadata: ", p), err) + } + if err := p.CryptoMetadata.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.CryptoMetadata), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 8:crypto_metadata: ", p), err) + } + } + return err } func (p *ColumnChunk) writeField9(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetEncryptedColumnMetadata() { - if err := oprot.WriteFieldBegin(ctx, "encrypted_column_metadata", thrift.STRING, 9); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 9:encrypted_column_metadata: ", p), err) } - if err := oprot.WriteBinary(ctx, p.EncryptedColumnMetadata); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.encrypted_column_metadata (9) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 9:encrypted_column_metadata: ", p), err) } - } - return err + if p.IsSetEncryptedColumnMetadata() { + if err := oprot.WriteFieldBegin(ctx, "encrypted_column_metadata", thrift.STRING, 9); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 9:encrypted_column_metadata: ", p), err) + } + if err := oprot.WriteBinary(ctx, p.EncryptedColumnMetadata); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.encrypted_column_metadata (9) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 9:encrypted_column_metadata: ", p), err) + } + } + return err } func (p *ColumnChunk) Equals(other *ColumnChunk) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if p.FilePath != other.FilePath { - if p.FilePath == nil || other.FilePath == nil { - return false - } - if (*p.FilePath) != (*other.FilePath) { return false } - } - if p.FileOffset != other.FileOffset { return false } - if !p.MetaData.Equals(other.MetaData) { return false } - if p.OffsetIndexOffset != other.OffsetIndexOffset { - if p.OffsetIndexOffset == nil || other.OffsetIndexOffset == nil { - return false - } - if (*p.OffsetIndexOffset) != (*other.OffsetIndexOffset) { return false } - } - if p.OffsetIndexLength != other.OffsetIndexLength { - if p.OffsetIndexLength == nil || other.OffsetIndexLength == nil { - return false - } - if (*p.OffsetIndexLength) != (*other.OffsetIndexLength) { return false } - } - if p.ColumnIndexOffset != other.ColumnIndexOffset { - if p.ColumnIndexOffset == nil || other.ColumnIndexOffset == nil { - return false - } - if (*p.ColumnIndexOffset) != (*other.ColumnIndexOffset) { return false } - } - if p.ColumnIndexLength != other.ColumnIndexLength { - if p.ColumnIndexLength == nil || other.ColumnIndexLength == nil { - return false - } - if (*p.ColumnIndexLength) != (*other.ColumnIndexLength) { return false } - } - if !p.CryptoMetadata.Equals(other.CryptoMetadata) { return false } - if bytes.Compare(p.EncryptedColumnMetadata, other.EncryptedColumnMetadata) != 0 { return false } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if p.FilePath != other.FilePath { + if p.FilePath == nil || other.FilePath == nil { + return false + } + if (*p.FilePath) != (*other.FilePath) { + return false + } + } + if p.FileOffset != other.FileOffset { + return false + } + if !p.MetaData.Equals(other.MetaData) { + return false + } + if p.OffsetIndexOffset != other.OffsetIndexOffset { + if p.OffsetIndexOffset == nil || other.OffsetIndexOffset == nil { + return false + } + if (*p.OffsetIndexOffset) != (*other.OffsetIndexOffset) { + return false + } + } + if p.OffsetIndexLength != other.OffsetIndexLength { + if p.OffsetIndexLength == nil || other.OffsetIndexLength == nil { + return false + } + if (*p.OffsetIndexLength) != (*other.OffsetIndexLength) { + return false + } + } + if p.ColumnIndexOffset != other.ColumnIndexOffset { + if p.ColumnIndexOffset == nil || other.ColumnIndexOffset == nil { + return false + } + if (*p.ColumnIndexOffset) != (*other.ColumnIndexOffset) { + return false + } + } + if p.ColumnIndexLength != other.ColumnIndexLength { + if p.ColumnIndexLength == nil || other.ColumnIndexLength == nil { + return false + } + if (*p.ColumnIndexLength) != (*other.ColumnIndexLength) { + return false + } + } + if !p.CryptoMetadata.Equals(other.CryptoMetadata) { + return false + } + if bytes.Compare(p.EncryptedColumnMetadata, other.EncryptedColumnMetadata) != 0 { + return false + } + return true } func (p *ColumnChunk) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("ColumnChunk(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("ColumnChunk(%+v)", *p) } func (p *ColumnChunk) Validate() error { - return nil + return nil } + // Attributes: -// - Columns: Metadata for each column chunk in this row group. +// - Columns: Metadata for each column chunk in this row group. +// // This list must have the same order as the SchemaElement list in FileMetaData. -// -// - TotalByteSize: Total byte size of all the uncompressed column data in this row group * -// - NumRows: Number of rows in this row group * -// - SortingColumns: If set, specifies a sort ordering of the rows in this RowGroup. +// +// - TotalByteSize: Total byte size of all the uncompressed column data in this row group * +// - NumRows: Number of rows in this row group * +// - SortingColumns: If set, specifies a sort ordering of the rows in this RowGroup. +// // The sorting columns can be a subset of all the columns. -// - FileOffset: Byte offset from beginning of file to first page (data or dictionary) +// - FileOffset: Byte offset from beginning of file to first page (data or dictionary) +// // in this row group * -// - TotalCompressedSize: Total byte size of all compressed (and potentially encrypted) column data +// - TotalCompressedSize: Total byte size of all compressed (and potentially encrypted) column data +// // in this row group * -// - Ordinal: Row group ordinal in the file * +// - Ordinal: Row group ordinal in the file * type RowGroup struct { - Columns []*ColumnChunk `thrift:"columns,1,required" db:"columns" json:"columns"` - TotalByteSize int64 `thrift:"total_byte_size,2,required" db:"total_byte_size" json:"total_byte_size"` - NumRows int64 `thrift:"num_rows,3,required" db:"num_rows" json:"num_rows"` - SortingColumns []*SortingColumn `thrift:"sorting_columns,4" db:"sorting_columns" json:"sorting_columns,omitempty"` - FileOffset *int64 `thrift:"file_offset,5" db:"file_offset" json:"file_offset,omitempty"` - TotalCompressedSize *int64 `thrift:"total_compressed_size,6" db:"total_compressed_size" json:"total_compressed_size,omitempty"` - Ordinal *int16 `thrift:"ordinal,7" db:"ordinal" json:"ordinal,omitempty"` + Columns []*ColumnChunk `thrift:"columns,1,required" db:"columns" json:"columns"` + TotalByteSize int64 `thrift:"total_byte_size,2,required" db:"total_byte_size" json:"total_byte_size"` + NumRows int64 `thrift:"num_rows,3,required" db:"num_rows" json:"num_rows"` + SortingColumns []*SortingColumn `thrift:"sorting_columns,4" db:"sorting_columns" json:"sorting_columns,omitempty"` + FileOffset *int64 `thrift:"file_offset,5" db:"file_offset" json:"file_offset,omitempty"` + TotalCompressedSize *int64 `thrift:"total_compressed_size,6" db:"total_compressed_size" json:"total_compressed_size,omitempty"` + Ordinal *int16 `thrift:"ordinal,7" db:"ordinal" json:"ordinal,omitempty"` } func NewRowGroup() *RowGroup { - return &RowGroup{} + return &RowGroup{} } - func (p *RowGroup) GetColumns() []*ColumnChunk { - return p.Columns + return p.Columns } func (p *RowGroup) GetTotalByteSize() int64 { - return p.TotalByteSize + return p.TotalByteSize } func (p *RowGroup) GetNumRows() int64 { - return p.NumRows + return p.NumRows } + var RowGroup_SortingColumns_DEFAULT []*SortingColumn func (p *RowGroup) GetSortingColumns() []*SortingColumn { - return p.SortingColumns + return p.SortingColumns } + var RowGroup_FileOffset_DEFAULT int64 + func (p *RowGroup) GetFileOffset() int64 { - if !p.IsSetFileOffset() { - return RowGroup_FileOffset_DEFAULT - } -return *p.FileOffset + if !p.IsSetFileOffset() { + return RowGroup_FileOffset_DEFAULT + } + return *p.FileOffset } + var RowGroup_TotalCompressedSize_DEFAULT int64 + func (p *RowGroup) GetTotalCompressedSize() int64 { - if !p.IsSetTotalCompressedSize() { - return RowGroup_TotalCompressedSize_DEFAULT - } -return *p.TotalCompressedSize + if !p.IsSetTotalCompressedSize() { + return RowGroup_TotalCompressedSize_DEFAULT + } + return *p.TotalCompressedSize } + var RowGroup_Ordinal_DEFAULT int16 + func (p *RowGroup) GetOrdinal() int16 { - if !p.IsSetOrdinal() { - return RowGroup_Ordinal_DEFAULT - } -return *p.Ordinal + if !p.IsSetOrdinal() { + return RowGroup_Ordinal_DEFAULT + } + return *p.Ordinal } func (p *RowGroup) IsSetSortingColumns() bool { - return p.SortingColumns != nil + return p.SortingColumns != nil } func (p *RowGroup) IsSetFileOffset() bool { - return p.FileOffset != nil + return p.FileOffset != nil } func (p *RowGroup) IsSetTotalCompressedSize() bool { - return p.TotalCompressedSize != nil + return p.TotalCompressedSize != nil } func (p *RowGroup) IsSetOrdinal() bool { - return p.Ordinal != nil + return p.Ordinal != nil } func (p *RowGroup) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - var issetColumns bool = false; - var issetTotalByteSize bool = false; - var issetNumRows bool = false; - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.LIST { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - issetColumns = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.I64 { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - issetTotalByteSize = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 3: - if fieldTypeId == thrift.I64 { - if err := p.ReadField3(ctx, iprot); err != nil { - return err - } - issetNumRows = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 4: - if fieldTypeId == thrift.LIST { - if err := p.ReadField4(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 5: - if fieldTypeId == thrift.I64 { - if err := p.ReadField5(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 6: - if fieldTypeId == thrift.I64 { - if err := p.ReadField6(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 7: - if fieldTypeId == thrift.I16 { - if err := p.ReadField7(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - if !issetColumns{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Columns is not set")); - } - if !issetTotalByteSize{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field TotalByteSize is not set")); - } - if !issetNumRows{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field NumRows is not set")); - } - return nil -} - -func (p *RowGroup) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - _, size, err := iprot.ReadListBegin(ctx) - if err != nil { - return thrift.PrependError("error reading list begin: ", err) - } - tSlice := make([]*ColumnChunk, 0, size) - p.Columns = tSlice - for i := 0; i < size; i ++ { - _elem10 := &ColumnChunk{} - if err := _elem10.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem10), err) - } - p.Columns = append(p.Columns, _elem10) - } - if err := iprot.ReadListEnd(ctx); err != nil { - return thrift.PrependError("error reading list end: ", err) - } - return nil -} - -func (p *RowGroup) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(ctx); err != nil { - return thrift.PrependError("error reading field 2: ", err) -} else { - p.TotalByteSize = v -} - return nil -} - -func (p *RowGroup) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(ctx); err != nil { - return thrift.PrependError("error reading field 3: ", err) -} else { - p.NumRows = v -} - return nil -} - -func (p *RowGroup) ReadField4(ctx context.Context, iprot thrift.TProtocol) error { - _, size, err := iprot.ReadListBegin(ctx) - if err != nil { - return thrift.PrependError("error reading list begin: ", err) - } - tSlice := make([]*SortingColumn, 0, size) - p.SortingColumns = tSlice - for i := 0; i < size; i ++ { - _elem11 := &SortingColumn{} - if err := _elem11.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem11), err) - } - p.SortingColumns = append(p.SortingColumns, _elem11) - } - if err := iprot.ReadListEnd(ctx); err != nil { - return thrift.PrependError("error reading list end: ", err) - } - return nil -} - -func (p *RowGroup) ReadField5(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(ctx); err != nil { - return thrift.PrependError("error reading field 5: ", err) -} else { - p.FileOffset = &v -} - return nil -} - -func (p *RowGroup) ReadField6(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(ctx); err != nil { - return thrift.PrependError("error reading field 6: ", err) -} else { - p.TotalCompressedSize = &v -} - return nil -} - -func (p *RowGroup) ReadField7(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI16(ctx); err != nil { - return thrift.PrependError("error reading field 7: ", err) -} else { - p.Ordinal = &v -} - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + var issetColumns bool = false + var issetTotalByteSize bool = false + var issetNumRows bool = false + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.LIST { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + issetColumns = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.I64 { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + issetTotalByteSize = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 3: + if fieldTypeId == thrift.I64 { + if err := p.ReadField3(ctx, iprot); err != nil { + return err + } + issetNumRows = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 4: + if fieldTypeId == thrift.LIST { + if err := p.ReadField4(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 5: + if fieldTypeId == thrift.I64 { + if err := p.ReadField5(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 6: + if fieldTypeId == thrift.I64 { + if err := p.ReadField6(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 7: + if fieldTypeId == thrift.I16 { + if err := p.ReadField7(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + if !issetColumns { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Columns is not set")) + } + if !issetTotalByteSize { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field TotalByteSize is not set")) + } + if !issetNumRows { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field NumRows is not set")) + } + return nil +} + +func (p *RowGroup) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + _, size, err := iprot.ReadListBegin(ctx) + if err != nil { + return thrift.PrependError("error reading list begin: ", err) + } + tSlice := make([]*ColumnChunk, 0, size) + p.Columns = tSlice + for i := 0; i < size; i++ { + _elem10 := &ColumnChunk{} + if err := _elem10.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem10), err) + } + p.Columns = append(p.Columns, _elem10) + } + if err := iprot.ReadListEnd(ctx); err != nil { + return thrift.PrependError("error reading list end: ", err) + } + return nil +} + +func (p *RowGroup) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI64(ctx); err != nil { + return thrift.PrependError("error reading field 2: ", err) + } else { + p.TotalByteSize = v + } + return nil +} + +func (p *RowGroup) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI64(ctx); err != nil { + return thrift.PrependError("error reading field 3: ", err) + } else { + p.NumRows = v + } + return nil +} + +func (p *RowGroup) ReadField4(ctx context.Context, iprot thrift.TProtocol) error { + _, size, err := iprot.ReadListBegin(ctx) + if err != nil { + return thrift.PrependError("error reading list begin: ", err) + } + tSlice := make([]*SortingColumn, 0, size) + p.SortingColumns = tSlice + for i := 0; i < size; i++ { + _elem11 := &SortingColumn{} + if err := _elem11.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem11), err) + } + p.SortingColumns = append(p.SortingColumns, _elem11) + } + if err := iprot.ReadListEnd(ctx); err != nil { + return thrift.PrependError("error reading list end: ", err) + } + return nil +} + +func (p *RowGroup) ReadField5(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI64(ctx); err != nil { + return thrift.PrependError("error reading field 5: ", err) + } else { + p.FileOffset = &v + } + return nil +} + +func (p *RowGroup) ReadField6(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI64(ctx); err != nil { + return thrift.PrependError("error reading field 6: ", err) + } else { + p.TotalCompressedSize = &v + } + return nil +} + +func (p *RowGroup) ReadField7(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI16(ctx); err != nil { + return thrift.PrependError("error reading field 7: ", err) + } else { + p.Ordinal = &v + } + return nil } func (p *RowGroup) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "RowGroup"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - if err := p.writeField3(ctx, oprot); err != nil { return err } - if err := p.writeField4(ctx, oprot); err != nil { return err } - if err := p.writeField5(ctx, oprot); err != nil { return err } - if err := p.writeField6(ctx, oprot); err != nil { return err } - if err := p.writeField7(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "RowGroup"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + if err := p.writeField3(ctx, oprot); err != nil { + return err + } + if err := p.writeField4(ctx, oprot); err != nil { + return err + } + if err := p.writeField5(ctx, oprot); err != nil { + return err + } + if err := p.writeField6(ctx, oprot); err != nil { + return err + } + if err := p.writeField7(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *RowGroup) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "columns", thrift.LIST, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:columns: ", p), err) } - if err := oprot.WriteListBegin(ctx, thrift.STRUCT, len(p.Columns)); err != nil { - return thrift.PrependError("error writing list begin: ", err) - } - for _, v := range p.Columns { - if err := v.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", v), err) - } - } - if err := oprot.WriteListEnd(ctx); err != nil { - return thrift.PrependError("error writing list end: ", err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:columns: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "columns", thrift.LIST, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:columns: ", p), err) + } + if err := oprot.WriteListBegin(ctx, thrift.STRUCT, len(p.Columns)); err != nil { + return thrift.PrependError("error writing list begin: ", err) + } + for _, v := range p.Columns { + if err := v.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", v), err) + } + } + if err := oprot.WriteListEnd(ctx); err != nil { + return thrift.PrependError("error writing list end: ", err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:columns: ", p), err) + } + return err } func (p *RowGroup) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "total_byte_size", thrift.I64, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:total_byte_size: ", p), err) } - if err := oprot.WriteI64(ctx, int64(p.TotalByteSize)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.total_byte_size (2) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:total_byte_size: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "total_byte_size", thrift.I64, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:total_byte_size: ", p), err) + } + if err := oprot.WriteI64(ctx, int64(p.TotalByteSize)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.total_byte_size (2) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:total_byte_size: ", p), err) + } + return err } func (p *RowGroup) writeField3(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "num_rows", thrift.I64, 3); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:num_rows: ", p), err) } - if err := oprot.WriteI64(ctx, int64(p.NumRows)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.num_rows (3) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 3:num_rows: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "num_rows", thrift.I64, 3); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:num_rows: ", p), err) + } + if err := oprot.WriteI64(ctx, int64(p.NumRows)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.num_rows (3) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 3:num_rows: ", p), err) + } + return err } func (p *RowGroup) writeField4(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetSortingColumns() { - if err := oprot.WriteFieldBegin(ctx, "sorting_columns", thrift.LIST, 4); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:sorting_columns: ", p), err) } - if err := oprot.WriteListBegin(ctx, thrift.STRUCT, len(p.SortingColumns)); err != nil { - return thrift.PrependError("error writing list begin: ", err) - } - for _, v := range p.SortingColumns { - if err := v.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", v), err) - } - } - if err := oprot.WriteListEnd(ctx); err != nil { - return thrift.PrependError("error writing list end: ", err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 4:sorting_columns: ", p), err) } - } - return err + if p.IsSetSortingColumns() { + if err := oprot.WriteFieldBegin(ctx, "sorting_columns", thrift.LIST, 4); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:sorting_columns: ", p), err) + } + if err := oprot.WriteListBegin(ctx, thrift.STRUCT, len(p.SortingColumns)); err != nil { + return thrift.PrependError("error writing list begin: ", err) + } + for _, v := range p.SortingColumns { + if err := v.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", v), err) + } + } + if err := oprot.WriteListEnd(ctx); err != nil { + return thrift.PrependError("error writing list end: ", err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 4:sorting_columns: ", p), err) + } + } + return err } func (p *RowGroup) writeField5(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetFileOffset() { - if err := oprot.WriteFieldBegin(ctx, "file_offset", thrift.I64, 5); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:file_offset: ", p), err) } - if err := oprot.WriteI64(ctx, int64(*p.FileOffset)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.file_offset (5) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 5:file_offset: ", p), err) } - } - return err + if p.IsSetFileOffset() { + if err := oprot.WriteFieldBegin(ctx, "file_offset", thrift.I64, 5); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:file_offset: ", p), err) + } + if err := oprot.WriteI64(ctx, int64(*p.FileOffset)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.file_offset (5) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 5:file_offset: ", p), err) + } + } + return err } func (p *RowGroup) writeField6(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetTotalCompressedSize() { - if err := oprot.WriteFieldBegin(ctx, "total_compressed_size", thrift.I64, 6); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 6:total_compressed_size: ", p), err) } - if err := oprot.WriteI64(ctx, int64(*p.TotalCompressedSize)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.total_compressed_size (6) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 6:total_compressed_size: ", p), err) } - } - return err + if p.IsSetTotalCompressedSize() { + if err := oprot.WriteFieldBegin(ctx, "total_compressed_size", thrift.I64, 6); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 6:total_compressed_size: ", p), err) + } + if err := oprot.WriteI64(ctx, int64(*p.TotalCompressedSize)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.total_compressed_size (6) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 6:total_compressed_size: ", p), err) + } + } + return err } func (p *RowGroup) writeField7(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetOrdinal() { - if err := oprot.WriteFieldBegin(ctx, "ordinal", thrift.I16, 7); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 7:ordinal: ", p), err) } - if err := oprot.WriteI16(ctx, int16(*p.Ordinal)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.ordinal (7) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 7:ordinal: ", p), err) } - } - return err + if p.IsSetOrdinal() { + if err := oprot.WriteFieldBegin(ctx, "ordinal", thrift.I16, 7); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 7:ordinal: ", p), err) + } + if err := oprot.WriteI16(ctx, int16(*p.Ordinal)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.ordinal (7) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 7:ordinal: ", p), err) + } + } + return err } func (p *RowGroup) Equals(other *RowGroup) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if len(p.Columns) != len(other.Columns) { return false } - for i, _tgt := range p.Columns { - _src12 := other.Columns[i] - if !_tgt.Equals(_src12) { return false } - } - if p.TotalByteSize != other.TotalByteSize { return false } - if p.NumRows != other.NumRows { return false } - if len(p.SortingColumns) != len(other.SortingColumns) { return false } - for i, _tgt := range p.SortingColumns { - _src13 := other.SortingColumns[i] - if !_tgt.Equals(_src13) { return false } - } - if p.FileOffset != other.FileOffset { - if p.FileOffset == nil || other.FileOffset == nil { - return false - } - if (*p.FileOffset) != (*other.FileOffset) { return false } - } - if p.TotalCompressedSize != other.TotalCompressedSize { - if p.TotalCompressedSize == nil || other.TotalCompressedSize == nil { - return false - } - if (*p.TotalCompressedSize) != (*other.TotalCompressedSize) { return false } - } - if p.Ordinal != other.Ordinal { - if p.Ordinal == nil || other.Ordinal == nil { - return false - } - if (*p.Ordinal) != (*other.Ordinal) { return false } - } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if len(p.Columns) != len(other.Columns) { + return false + } + for i, _tgt := range p.Columns { + _src12 := other.Columns[i] + if !_tgt.Equals(_src12) { + return false + } + } + if p.TotalByteSize != other.TotalByteSize { + return false + } + if p.NumRows != other.NumRows { + return false + } + if len(p.SortingColumns) != len(other.SortingColumns) { + return false + } + for i, _tgt := range p.SortingColumns { + _src13 := other.SortingColumns[i] + if !_tgt.Equals(_src13) { + return false + } + } + if p.FileOffset != other.FileOffset { + if p.FileOffset == nil || other.FileOffset == nil { + return false + } + if (*p.FileOffset) != (*other.FileOffset) { + return false + } + } + if p.TotalCompressedSize != other.TotalCompressedSize { + if p.TotalCompressedSize == nil || other.TotalCompressedSize == nil { + return false + } + if (*p.TotalCompressedSize) != (*other.TotalCompressedSize) { + return false + } + } + if p.Ordinal != other.Ordinal { + if p.Ordinal == nil || other.Ordinal == nil { + return false + } + if (*p.Ordinal) != (*other.Ordinal) { + return false + } + } + return true } func (p *RowGroup) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("RowGroup(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("RowGroup(%+v)", *p) } func (p *RowGroup) Validate() error { - return nil + return nil } + // Empty struct to signal the order defined by the physical or logical type type TypeDefinedOrder struct { } func NewTypeDefinedOrder() *TypeDefinedOrder { - return &TypeDefinedOrder{} + return &TypeDefinedOrder{} } func (p *TypeDefinedOrder) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil } func (p *TypeDefinedOrder) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "TypeDefinedOrder"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "TypeDefinedOrder"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *TypeDefinedOrder) Equals(other *TypeDefinedOrder) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + return true } func (p *TypeDefinedOrder) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("TypeDefinedOrder(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("TypeDefinedOrder(%+v)", *p) } func (p *TypeDefinedOrder) Validate() error { - return nil + return nil } + // Union to specify the order used for the min_value and max_value fields for a // column. This union takes the role of an enhanced enum that allows rich // elements (which will be needed for a collation-based ordering in the future). -// +// // Possible values are: -// * TypeDefinedOrder - the column uses the order defined by its logical or -// physical type (if there is no logical type). -// +// - TypeDefinedOrder - the column uses the order defined by its logical or +// physical type (if there is no logical type). +// // If the reader does not support the value of this union, min and max stats // for this column should be ignored. -// +// // Attributes: -// - TYPE_ORDER: The sort orders for logical types are: -// UTF8 - unsigned byte-wise comparison -// INT8 - signed comparison -// INT16 - signed comparison -// INT32 - signed comparison -// INT64 - signed comparison -// UINT8 - unsigned comparison -// UINT16 - unsigned comparison -// UINT32 - unsigned comparison -// UINT64 - unsigned comparison -// DECIMAL - signed comparison of the represented value -// DATE - signed comparison -// TIME_MILLIS - signed comparison -// TIME_MICROS - signed comparison -// TIMESTAMP_MILLIS - signed comparison -// TIMESTAMP_MICROS - signed comparison -// INTERVAL - unsigned comparison -// JSON - unsigned byte-wise comparison -// BSON - unsigned byte-wise comparison -// ENUM - unsigned byte-wise comparison -// LIST - undefined -// MAP - undefined -// +// - TYPE_ORDER: The sort orders for logical types are: +// UTF8 - unsigned byte-wise comparison +// INT8 - signed comparison +// INT16 - signed comparison +// INT32 - signed comparison +// INT64 - signed comparison +// UINT8 - unsigned comparison +// UINT16 - unsigned comparison +// UINT32 - unsigned comparison +// UINT64 - unsigned comparison +// DECIMAL - signed comparison of the represented value +// DATE - signed comparison +// TIME_MILLIS - signed comparison +// TIME_MICROS - signed comparison +// TIMESTAMP_MILLIS - signed comparison +// TIMESTAMP_MICROS - signed comparison +// INTERVAL - unsigned comparison +// JSON - unsigned byte-wise comparison +// BSON - unsigned byte-wise comparison +// ENUM - unsigned byte-wise comparison +// LIST - undefined +// MAP - undefined +// // In the absence of logical types, the sort order is determined by the physical type: -// BOOLEAN - false, true -// INT32 - signed comparison -// INT64 - signed comparison -// INT96 (only used for legacy timestamps) - undefined -// FLOAT - signed comparison of the represented value (*) -// DOUBLE - signed comparison of the represented value (*) -// BYTE_ARRAY - unsigned byte-wise comparison -// FIXED_LEN_BYTE_ARRAY - unsigned byte-wise comparison -// +// +// BOOLEAN - false, true +// INT32 - signed comparison +// INT64 - signed comparison +// INT96 (only used for legacy timestamps) - undefined +// FLOAT - signed comparison of the represented value (*) +// DOUBLE - signed comparison of the represented value (*) +// BYTE_ARRAY - unsigned byte-wise comparison +// FIXED_LEN_BYTE_ARRAY - unsigned byte-wise comparison +// // (*) Because the sorting order is not specified properly for floating -// point values (relations vs. total ordering) the following -// compatibility rules should be applied when reading statistics: -// - If the min is a NaN, it should be ignored. -// - If the max is a NaN, it should be ignored. -// - If the min is +0, the row group may contain -0 values as well. -// - If the max is -0, the row group may contain +0 values as well. -// - When looking for NaN values, min and max should be ignored. -// -// When writing statistics the following rules should be followed: -// - NaNs should not be written to min or max statistics fields. -// - If the computed max value is zero (whether negative or positive), -// `+0.0` should be written into the max statistics field. -// - If the computed min value is zero (whether negative or positive), -// `-0.0` should be written into the min statistics field. +// +// point values (relations vs. total ordering) the following +// compatibility rules should be applied when reading statistics: +// - If the min is a NaN, it should be ignored. +// - If the max is a NaN, it should be ignored. +// - If the min is +0, the row group may contain -0 values as well. +// - If the max is -0, the row group may contain +0 values as well. +// - When looking for NaN values, min and max should be ignored. +// +// When writing statistics the following rules should be followed: +// - NaNs should not be written to min or max statistics fields. +// - If the computed max value is zero (whether negative or positive), +// `+0.0` should be written into the max statistics field. +// - If the computed min value is zero (whether negative or positive), +// `-0.0` should be written into the min statistics field. type ColumnOrder struct { - TYPE_ORDER *TypeDefinedOrder `thrift:"TYPE_ORDER,1" db:"TYPE_ORDER" json:"TYPE_ORDER,omitempty"` + TYPE_ORDER *TypeDefinedOrder `thrift:"TYPE_ORDER,1" db:"TYPE_ORDER" json:"TYPE_ORDER,omitempty"` } func NewColumnOrder() *ColumnOrder { - return &ColumnOrder{} + return &ColumnOrder{} } var ColumnOrder_TYPE_ORDER_DEFAULT *TypeDefinedOrder + func (p *ColumnOrder) GetTYPE_ORDER() *TypeDefinedOrder { - if !p.IsSetTYPE_ORDER() { - return ColumnOrder_TYPE_ORDER_DEFAULT - } -return p.TYPE_ORDER + if !p.IsSetTYPE_ORDER() { + return ColumnOrder_TYPE_ORDER_DEFAULT + } + return p.TYPE_ORDER } func (p *ColumnOrder) CountSetFieldsColumnOrder() int { - count := 0 - if (p.IsSetTYPE_ORDER()) { - count++ - } - return count + count := 0 + if p.IsSetTYPE_ORDER() { + count++ + } + return count } func (p *ColumnOrder) IsSetTYPE_ORDER() bool { - return p.TYPE_ORDER != nil + return p.TYPE_ORDER != nil } func (p *ColumnOrder) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil -} - -func (p *ColumnOrder) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - p.TYPE_ORDER = &TypeDefinedOrder{} - if err := p.TYPE_ORDER.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.TYPE_ORDER), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil +} + +func (p *ColumnOrder) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + p.TYPE_ORDER = &TypeDefinedOrder{} + if err := p.TYPE_ORDER.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.TYPE_ORDER), err) + } + return nil } func (p *ColumnOrder) Write(ctx context.Context, oprot thrift.TProtocol) error { - if c := p.CountSetFieldsColumnOrder(); c != 1 { - return fmt.Errorf("%T write union: exactly one field must be set (%d set)", p, c) - } - if err := oprot.WriteStructBegin(ctx, "ColumnOrder"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if c := p.CountSetFieldsColumnOrder(); c != 1 { + return fmt.Errorf("%T write union: exactly one field must be set (%d set)", p, c) + } + if err := oprot.WriteStructBegin(ctx, "ColumnOrder"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *ColumnOrder) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetTYPE_ORDER() { - if err := oprot.WriteFieldBegin(ctx, "TYPE_ORDER", thrift.STRUCT, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:TYPE_ORDER: ", p), err) } - if err := p.TYPE_ORDER.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.TYPE_ORDER), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:TYPE_ORDER: ", p), err) } - } - return err + if p.IsSetTYPE_ORDER() { + if err := oprot.WriteFieldBegin(ctx, "TYPE_ORDER", thrift.STRUCT, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:TYPE_ORDER: ", p), err) + } + if err := p.TYPE_ORDER.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.TYPE_ORDER), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:TYPE_ORDER: ", p), err) + } + } + return err } func (p *ColumnOrder) Equals(other *ColumnOrder) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if !p.TYPE_ORDER.Equals(other.TYPE_ORDER) { return false } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if !p.TYPE_ORDER.Equals(other.TYPE_ORDER) { + return false + } + return true } func (p *ColumnOrder) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("ColumnOrder(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("ColumnOrder(%+v)", *p) } func (p *ColumnOrder) Validate() error { - return nil + return nil } + // Attributes: -// - Offset: Offset of the page in the file * -// - CompressedPageSize: Size of the page, including header. Sum of compressed_page_size and header +// - Offset: Offset of the page in the file * +// - CompressedPageSize: Size of the page, including header. Sum of compressed_page_size and header +// // length -// - FirstRowIndex: Index within the RowGroup of the first row of the page; this means pages +// - FirstRowIndex: Index within the RowGroup of the first row of the page; this means pages +// // change on record boundaries (r = 0). type PageLocation struct { - Offset int64 `thrift:"offset,1,required" db:"offset" json:"offset"` - CompressedPageSize int32 `thrift:"compressed_page_size,2,required" db:"compressed_page_size" json:"compressed_page_size"` - FirstRowIndex int64 `thrift:"first_row_index,3,required" db:"first_row_index" json:"first_row_index"` + Offset int64 `thrift:"offset,1,required" db:"offset" json:"offset"` + CompressedPageSize int32 `thrift:"compressed_page_size,2,required" db:"compressed_page_size" json:"compressed_page_size"` + FirstRowIndex int64 `thrift:"first_row_index,3,required" db:"first_row_index" json:"first_row_index"` } func NewPageLocation() *PageLocation { - return &PageLocation{} + return &PageLocation{} } - func (p *PageLocation) GetOffset() int64 { - return p.Offset + return p.Offset } func (p *PageLocation) GetCompressedPageSize() int32 { - return p.CompressedPageSize + return p.CompressedPageSize } func (p *PageLocation) GetFirstRowIndex() int64 { - return p.FirstRowIndex + return p.FirstRowIndex } func (p *PageLocation) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - var issetOffset bool = false; - var issetCompressedPageSize bool = false; - var issetFirstRowIndex bool = false; - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.I64 { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - issetOffset = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.I32 { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - issetCompressedPageSize = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 3: - if fieldTypeId == thrift.I64 { - if err := p.ReadField3(ctx, iprot); err != nil { - return err - } - issetFirstRowIndex = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - if !issetOffset{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Offset is not set")); - } - if !issetCompressedPageSize{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field CompressedPageSize is not set")); - } - if !issetFirstRowIndex{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field FirstRowIndex is not set")); - } - return nil -} - -func (p *PageLocation) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(ctx); err != nil { - return thrift.PrependError("error reading field 1: ", err) -} else { - p.Offset = v -} - return nil -} - -func (p *PageLocation) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 2: ", err) -} else { - p.CompressedPageSize = v -} - return nil -} - -func (p *PageLocation) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(ctx); err != nil { - return thrift.PrependError("error reading field 3: ", err) -} else { - p.FirstRowIndex = v -} - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + var issetOffset bool = false + var issetCompressedPageSize bool = false + var issetFirstRowIndex bool = false + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.I64 { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + issetOffset = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.I32 { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + issetCompressedPageSize = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 3: + if fieldTypeId == thrift.I64 { + if err := p.ReadField3(ctx, iprot); err != nil { + return err + } + issetFirstRowIndex = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + if !issetOffset { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Offset is not set")) + } + if !issetCompressedPageSize { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field CompressedPageSize is not set")) + } + if !issetFirstRowIndex { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field FirstRowIndex is not set")) + } + return nil +} + +func (p *PageLocation) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI64(ctx); err != nil { + return thrift.PrependError("error reading field 1: ", err) + } else { + p.Offset = v + } + return nil +} + +func (p *PageLocation) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 2: ", err) + } else { + p.CompressedPageSize = v + } + return nil +} + +func (p *PageLocation) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI64(ctx); err != nil { + return thrift.PrependError("error reading field 3: ", err) + } else { + p.FirstRowIndex = v + } + return nil } func (p *PageLocation) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "PageLocation"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - if err := p.writeField3(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "PageLocation"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + if err := p.writeField3(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *PageLocation) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "offset", thrift.I64, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:offset: ", p), err) } - if err := oprot.WriteI64(ctx, int64(p.Offset)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.offset (1) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:offset: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "offset", thrift.I64, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:offset: ", p), err) + } + if err := oprot.WriteI64(ctx, int64(p.Offset)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.offset (1) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:offset: ", p), err) + } + return err } func (p *PageLocation) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "compressed_page_size", thrift.I32, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:compressed_page_size: ", p), err) } - if err := oprot.WriteI32(ctx, int32(p.CompressedPageSize)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.compressed_page_size (2) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:compressed_page_size: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "compressed_page_size", thrift.I32, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:compressed_page_size: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.CompressedPageSize)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.compressed_page_size (2) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:compressed_page_size: ", p), err) + } + return err } func (p *PageLocation) writeField3(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "first_row_index", thrift.I64, 3); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:first_row_index: ", p), err) } - if err := oprot.WriteI64(ctx, int64(p.FirstRowIndex)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.first_row_index (3) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 3:first_row_index: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "first_row_index", thrift.I64, 3); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:first_row_index: ", p), err) + } + if err := oprot.WriteI64(ctx, int64(p.FirstRowIndex)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.first_row_index (3) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 3:first_row_index: ", p), err) + } + return err } func (p *PageLocation) Equals(other *PageLocation) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if p.Offset != other.Offset { return false } - if p.CompressedPageSize != other.CompressedPageSize { return false } - if p.FirstRowIndex != other.FirstRowIndex { return false } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if p.Offset != other.Offset { + return false + } + if p.CompressedPageSize != other.CompressedPageSize { + return false + } + if p.FirstRowIndex != other.FirstRowIndex { + return false + } + return true } func (p *PageLocation) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("PageLocation(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("PageLocation(%+v)", *p) } func (p *PageLocation) Validate() error { - return nil + return nil } + // Attributes: -// - PageLocations: PageLocations, ordered by increasing PageLocation.offset. It is required +// - PageLocations: PageLocations, ordered by increasing PageLocation.offset. It is required +// // that page_locations[i].first_row_index < page_locations[i+1].first_row_index. type OffsetIndex struct { - PageLocations []*PageLocation `thrift:"page_locations,1,required" db:"page_locations" json:"page_locations"` + PageLocations []*PageLocation `thrift:"page_locations,1,required" db:"page_locations" json:"page_locations"` } func NewOffsetIndex() *OffsetIndex { - return &OffsetIndex{} + return &OffsetIndex{} } - func (p *OffsetIndex) GetPageLocations() []*PageLocation { - return p.PageLocations + return p.PageLocations } func (p *OffsetIndex) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - var issetPageLocations bool = false; - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.LIST { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - issetPageLocations = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - if !issetPageLocations{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field PageLocations is not set")); - } - return nil -} - -func (p *OffsetIndex) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - _, size, err := iprot.ReadListBegin(ctx) - if err != nil { - return thrift.PrependError("error reading list begin: ", err) - } - tSlice := make([]*PageLocation, 0, size) - p.PageLocations = tSlice - for i := 0; i < size; i ++ { - _elem14 := &PageLocation{} - if err := _elem14.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem14), err) - } - p.PageLocations = append(p.PageLocations, _elem14) - } - if err := iprot.ReadListEnd(ctx); err != nil { - return thrift.PrependError("error reading list end: ", err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + var issetPageLocations bool = false + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.LIST { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + issetPageLocations = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + if !issetPageLocations { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field PageLocations is not set")) + } + return nil +} + +func (p *OffsetIndex) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + _, size, err := iprot.ReadListBegin(ctx) + if err != nil { + return thrift.PrependError("error reading list begin: ", err) + } + tSlice := make([]*PageLocation, 0, size) + p.PageLocations = tSlice + for i := 0; i < size; i++ { + _elem14 := &PageLocation{} + if err := _elem14.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem14), err) + } + p.PageLocations = append(p.PageLocations, _elem14) + } + if err := iprot.ReadListEnd(ctx); err != nil { + return thrift.PrependError("error reading list end: ", err) + } + return nil } func (p *OffsetIndex) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "OffsetIndex"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "OffsetIndex"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *OffsetIndex) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "page_locations", thrift.LIST, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:page_locations: ", p), err) } - if err := oprot.WriteListBegin(ctx, thrift.STRUCT, len(p.PageLocations)); err != nil { - return thrift.PrependError("error writing list begin: ", err) - } - for _, v := range p.PageLocations { - if err := v.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", v), err) - } - } - if err := oprot.WriteListEnd(ctx); err != nil { - return thrift.PrependError("error writing list end: ", err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:page_locations: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "page_locations", thrift.LIST, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:page_locations: ", p), err) + } + if err := oprot.WriteListBegin(ctx, thrift.STRUCT, len(p.PageLocations)); err != nil { + return thrift.PrependError("error writing list begin: ", err) + } + for _, v := range p.PageLocations { + if err := v.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", v), err) + } + } + if err := oprot.WriteListEnd(ctx); err != nil { + return thrift.PrependError("error writing list end: ", err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:page_locations: ", p), err) + } + return err } func (p *OffsetIndex) Equals(other *OffsetIndex) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if len(p.PageLocations) != len(other.PageLocations) { return false } - for i, _tgt := range p.PageLocations { - _src15 := other.PageLocations[i] - if !_tgt.Equals(_src15) { return false } - } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if len(p.PageLocations) != len(other.PageLocations) { + return false + } + for i, _tgt := range p.PageLocations { + _src15 := other.PageLocations[i] + if !_tgt.Equals(_src15) { + return false + } + } + return true } func (p *OffsetIndex) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("OffsetIndex(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("OffsetIndex(%+v)", *p) } func (p *OffsetIndex) Validate() error { - return nil + return nil } + // Description for ColumnIndex. // Each [i] refers to the page at OffsetIndex.page_locations[i] -// +// // Attributes: -// - NullPages: A list of Boolean values to determine the validity of the corresponding +// - NullPages: A list of Boolean values to determine the validity of the corresponding +// // min and max values. If true, a page contains only null values, and writers // have to set the corresponding entries in min_values and max_values to // byte[0], so that all lists have the same length. If false, the // corresponding entries in min_values and max_values must be valid. -// - MinValues: Two lists containing lower and upper bounds for the values of each page +// - MinValues: Two lists containing lower and upper bounds for the values of each page +// // determined by the ColumnOrder of the column. These may be the actual // minimum and maximum values found on a page, but can also be (more compact) // values that do not exist on a page. For example, instead of storing ""Blart @@ -9528,1722 +10840,1957 @@ func (p *OffsetIndex) Validate() error { // Such more compact values must still be valid values within the column's // logical type. Readers must make sure that list entries are populated before // using them by inspecting null_pages. -// - MaxValues -// - BoundaryOrder: Stores whether both min_values and max_values are ordered and if so, in +// - MaxValues +// - BoundaryOrder: Stores whether both min_values and max_values are ordered and if so, in +// // which direction. This allows readers to perform binary searches in both // lists. Readers cannot assume that max_values[i] <= min_values[i+1], even // if the lists are ordered. -// - NullCounts: A list containing the number of null values for each page * +// - NullCounts: A list containing the number of null values for each page * type ColumnIndex struct { - NullPages []bool `thrift:"null_pages,1,required" db:"null_pages" json:"null_pages"` - MinValues [][]byte `thrift:"min_values,2,required" db:"min_values" json:"min_values"` - MaxValues [][]byte `thrift:"max_values,3,required" db:"max_values" json:"max_values"` - BoundaryOrder BoundaryOrder `thrift:"boundary_order,4,required" db:"boundary_order" json:"boundary_order"` - NullCounts []int64 `thrift:"null_counts,5" db:"null_counts" json:"null_counts,omitempty"` + NullPages []bool `thrift:"null_pages,1,required" db:"null_pages" json:"null_pages"` + MinValues [][]byte `thrift:"min_values,2,required" db:"min_values" json:"min_values"` + MaxValues [][]byte `thrift:"max_values,3,required" db:"max_values" json:"max_values"` + BoundaryOrder BoundaryOrder `thrift:"boundary_order,4,required" db:"boundary_order" json:"boundary_order"` + NullCounts []int64 `thrift:"null_counts,5" db:"null_counts" json:"null_counts,omitempty"` } func NewColumnIndex() *ColumnIndex { - return &ColumnIndex{} + return &ColumnIndex{} } - func (p *ColumnIndex) GetNullPages() []bool { - return p.NullPages + return p.NullPages } func (p *ColumnIndex) GetMinValues() [][]byte { - return p.MinValues + return p.MinValues } func (p *ColumnIndex) GetMaxValues() [][]byte { - return p.MaxValues + return p.MaxValues } func (p *ColumnIndex) GetBoundaryOrder() BoundaryOrder { - return p.BoundaryOrder + return p.BoundaryOrder } + var ColumnIndex_NullCounts_DEFAULT []int64 func (p *ColumnIndex) GetNullCounts() []int64 { - return p.NullCounts + return p.NullCounts } func (p *ColumnIndex) IsSetNullCounts() bool { - return p.NullCounts != nil + return p.NullCounts != nil } func (p *ColumnIndex) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - var issetNullPages bool = false; - var issetMinValues bool = false; - var issetMaxValues bool = false; - var issetBoundaryOrder bool = false; - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.LIST { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - issetNullPages = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.LIST { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - issetMinValues = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 3: - if fieldTypeId == thrift.LIST { - if err := p.ReadField3(ctx, iprot); err != nil { - return err - } - issetMaxValues = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 4: - if fieldTypeId == thrift.I32 { - if err := p.ReadField4(ctx, iprot); err != nil { - return err - } - issetBoundaryOrder = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 5: - if fieldTypeId == thrift.LIST { - if err := p.ReadField5(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - if !issetNullPages{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field NullPages is not set")); - } - if !issetMinValues{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field MinValues is not set")); - } - if !issetMaxValues{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field MaxValues is not set")); - } - if !issetBoundaryOrder{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field BoundaryOrder is not set")); - } - return nil -} - -func (p *ColumnIndex) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - _, size, err := iprot.ReadListBegin(ctx) - if err != nil { - return thrift.PrependError("error reading list begin: ", err) - } - tSlice := make([]bool, 0, size) - p.NullPages = tSlice - for i := 0; i < size; i ++ { -var _elem16 bool - if v, err := iprot.ReadBool(ctx); err != nil { - return thrift.PrependError("error reading field 0: ", err) -} else { - _elem16 = v -} - p.NullPages = append(p.NullPages, _elem16) - } - if err := iprot.ReadListEnd(ctx); err != nil { - return thrift.PrependError("error reading list end: ", err) - } - return nil -} - -func (p *ColumnIndex) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - _, size, err := iprot.ReadListBegin(ctx) - if err != nil { - return thrift.PrependError("error reading list begin: ", err) - } - tSlice := make([][]byte, 0, size) - p.MinValues = tSlice - for i := 0; i < size; i ++ { -var _elem17 []byte - if v, err := iprot.ReadBinary(ctx); err != nil { - return thrift.PrependError("error reading field 0: ", err) -} else { - _elem17 = v -} - p.MinValues = append(p.MinValues, _elem17) - } - if err := iprot.ReadListEnd(ctx); err != nil { - return thrift.PrependError("error reading list end: ", err) - } - return nil -} - -func (p *ColumnIndex) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { - _, size, err := iprot.ReadListBegin(ctx) - if err != nil { - return thrift.PrependError("error reading list begin: ", err) - } - tSlice := make([][]byte, 0, size) - p.MaxValues = tSlice - for i := 0; i < size; i ++ { -var _elem18 []byte - if v, err := iprot.ReadBinary(ctx); err != nil { - return thrift.PrependError("error reading field 0: ", err) -} else { - _elem18 = v -} - p.MaxValues = append(p.MaxValues, _elem18) - } - if err := iprot.ReadListEnd(ctx); err != nil { - return thrift.PrependError("error reading list end: ", err) - } - return nil -} - -func (p *ColumnIndex) ReadField4(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 4: ", err) -} else { - temp := BoundaryOrder(v) - p.BoundaryOrder = temp -} - return nil -} - -func (p *ColumnIndex) ReadField5(ctx context.Context, iprot thrift.TProtocol) error { - _, size, err := iprot.ReadListBegin(ctx) - if err != nil { - return thrift.PrependError("error reading list begin: ", err) - } - tSlice := make([]int64, 0, size) - p.NullCounts = tSlice - for i := 0; i < size; i ++ { -var _elem19 int64 - if v, err := iprot.ReadI64(ctx); err != nil { - return thrift.PrependError("error reading field 0: ", err) -} else { - _elem19 = v -} - p.NullCounts = append(p.NullCounts, _elem19) - } - if err := iprot.ReadListEnd(ctx); err != nil { - return thrift.PrependError("error reading list end: ", err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + var issetNullPages bool = false + var issetMinValues bool = false + var issetMaxValues bool = false + var issetBoundaryOrder bool = false + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.LIST { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + issetNullPages = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.LIST { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + issetMinValues = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 3: + if fieldTypeId == thrift.LIST { + if err := p.ReadField3(ctx, iprot); err != nil { + return err + } + issetMaxValues = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 4: + if fieldTypeId == thrift.I32 { + if err := p.ReadField4(ctx, iprot); err != nil { + return err + } + issetBoundaryOrder = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 5: + if fieldTypeId == thrift.LIST { + if err := p.ReadField5(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + if !issetNullPages { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field NullPages is not set")) + } + if !issetMinValues { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field MinValues is not set")) + } + if !issetMaxValues { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field MaxValues is not set")) + } + if !issetBoundaryOrder { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field BoundaryOrder is not set")) + } + return nil +} + +func (p *ColumnIndex) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + _, size, err := iprot.ReadListBegin(ctx) + if err != nil { + return thrift.PrependError("error reading list begin: ", err) + } + tSlice := make([]bool, 0, size) + p.NullPages = tSlice + for i := 0; i < size; i++ { + var _elem16 bool + if v, err := iprot.ReadBool(ctx); err != nil { + return thrift.PrependError("error reading field 0: ", err) + } else { + _elem16 = v + } + p.NullPages = append(p.NullPages, _elem16) + } + if err := iprot.ReadListEnd(ctx); err != nil { + return thrift.PrependError("error reading list end: ", err) + } + return nil +} + +func (p *ColumnIndex) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + _, size, err := iprot.ReadListBegin(ctx) + if err != nil { + return thrift.PrependError("error reading list begin: ", err) + } + tSlice := make([][]byte, 0, size) + p.MinValues = tSlice + for i := 0; i < size; i++ { + var _elem17 []byte + if v, err := iprot.ReadBinary(ctx); err != nil { + return thrift.PrependError("error reading field 0: ", err) + } else { + _elem17 = v + } + p.MinValues = append(p.MinValues, _elem17) + } + if err := iprot.ReadListEnd(ctx); err != nil { + return thrift.PrependError("error reading list end: ", err) + } + return nil +} + +func (p *ColumnIndex) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { + _, size, err := iprot.ReadListBegin(ctx) + if err != nil { + return thrift.PrependError("error reading list begin: ", err) + } + tSlice := make([][]byte, 0, size) + p.MaxValues = tSlice + for i := 0; i < size; i++ { + var _elem18 []byte + if v, err := iprot.ReadBinary(ctx); err != nil { + return thrift.PrependError("error reading field 0: ", err) + } else { + _elem18 = v + } + p.MaxValues = append(p.MaxValues, _elem18) + } + if err := iprot.ReadListEnd(ctx); err != nil { + return thrift.PrependError("error reading list end: ", err) + } + return nil +} + +func (p *ColumnIndex) ReadField4(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 4: ", err) + } else { + temp := BoundaryOrder(v) + p.BoundaryOrder = temp + } + return nil +} + +func (p *ColumnIndex) ReadField5(ctx context.Context, iprot thrift.TProtocol) error { + _, size, err := iprot.ReadListBegin(ctx) + if err != nil { + return thrift.PrependError("error reading list begin: ", err) + } + tSlice := make([]int64, 0, size) + p.NullCounts = tSlice + for i := 0; i < size; i++ { + var _elem19 int64 + if v, err := iprot.ReadI64(ctx); err != nil { + return thrift.PrependError("error reading field 0: ", err) + } else { + _elem19 = v + } + p.NullCounts = append(p.NullCounts, _elem19) + } + if err := iprot.ReadListEnd(ctx); err != nil { + return thrift.PrependError("error reading list end: ", err) + } + return nil } func (p *ColumnIndex) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "ColumnIndex"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - if err := p.writeField3(ctx, oprot); err != nil { return err } - if err := p.writeField4(ctx, oprot); err != nil { return err } - if err := p.writeField5(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "ColumnIndex"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + if err := p.writeField3(ctx, oprot); err != nil { + return err + } + if err := p.writeField4(ctx, oprot); err != nil { + return err + } + if err := p.writeField5(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *ColumnIndex) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "null_pages", thrift.LIST, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:null_pages: ", p), err) } - if err := oprot.WriteListBegin(ctx, thrift.BOOL, len(p.NullPages)); err != nil { - return thrift.PrependError("error writing list begin: ", err) - } - for _, v := range p.NullPages { - if err := oprot.WriteBool(ctx, bool(v)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T. (0) field write error: ", p), err) } - } - if err := oprot.WriteListEnd(ctx); err != nil { - return thrift.PrependError("error writing list end: ", err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:null_pages: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "null_pages", thrift.LIST, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:null_pages: ", p), err) + } + if err := oprot.WriteListBegin(ctx, thrift.BOOL, len(p.NullPages)); err != nil { + return thrift.PrependError("error writing list begin: ", err) + } + for _, v := range p.NullPages { + if err := oprot.WriteBool(ctx, bool(v)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T. (0) field write error: ", p), err) + } + } + if err := oprot.WriteListEnd(ctx); err != nil { + return thrift.PrependError("error writing list end: ", err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:null_pages: ", p), err) + } + return err } func (p *ColumnIndex) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "min_values", thrift.LIST, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:min_values: ", p), err) } - if err := oprot.WriteListBegin(ctx, thrift.STRING, len(p.MinValues)); err != nil { - return thrift.PrependError("error writing list begin: ", err) - } - for _, v := range p.MinValues { - if err := oprot.WriteBinary(ctx, v); err != nil { - return thrift.PrependError(fmt.Sprintf("%T. (0) field write error: ", p), err) } - } - if err := oprot.WriteListEnd(ctx); err != nil { - return thrift.PrependError("error writing list end: ", err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:min_values: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "min_values", thrift.LIST, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:min_values: ", p), err) + } + if err := oprot.WriteListBegin(ctx, thrift.STRING, len(p.MinValues)); err != nil { + return thrift.PrependError("error writing list begin: ", err) + } + for _, v := range p.MinValues { + if err := oprot.WriteBinary(ctx, v); err != nil { + return thrift.PrependError(fmt.Sprintf("%T. (0) field write error: ", p), err) + } + } + if err := oprot.WriteListEnd(ctx); err != nil { + return thrift.PrependError("error writing list end: ", err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:min_values: ", p), err) + } + return err } func (p *ColumnIndex) writeField3(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "max_values", thrift.LIST, 3); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:max_values: ", p), err) } - if err := oprot.WriteListBegin(ctx, thrift.STRING, len(p.MaxValues)); err != nil { - return thrift.PrependError("error writing list begin: ", err) - } - for _, v := range p.MaxValues { - if err := oprot.WriteBinary(ctx, v); err != nil { - return thrift.PrependError(fmt.Sprintf("%T. (0) field write error: ", p), err) } - } - if err := oprot.WriteListEnd(ctx); err != nil { - return thrift.PrependError("error writing list end: ", err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 3:max_values: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "max_values", thrift.LIST, 3); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:max_values: ", p), err) + } + if err := oprot.WriteListBegin(ctx, thrift.STRING, len(p.MaxValues)); err != nil { + return thrift.PrependError("error writing list begin: ", err) + } + for _, v := range p.MaxValues { + if err := oprot.WriteBinary(ctx, v); err != nil { + return thrift.PrependError(fmt.Sprintf("%T. (0) field write error: ", p), err) + } + } + if err := oprot.WriteListEnd(ctx); err != nil { + return thrift.PrependError("error writing list end: ", err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 3:max_values: ", p), err) + } + return err } func (p *ColumnIndex) writeField4(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "boundary_order", thrift.I32, 4); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:boundary_order: ", p), err) } - if err := oprot.WriteI32(ctx, int32(p.BoundaryOrder)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.boundary_order (4) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 4:boundary_order: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "boundary_order", thrift.I32, 4); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:boundary_order: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.BoundaryOrder)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.boundary_order (4) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 4:boundary_order: ", p), err) + } + return err } func (p *ColumnIndex) writeField5(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetNullCounts() { - if err := oprot.WriteFieldBegin(ctx, "null_counts", thrift.LIST, 5); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:null_counts: ", p), err) } - if err := oprot.WriteListBegin(ctx, thrift.I64, len(p.NullCounts)); err != nil { - return thrift.PrependError("error writing list begin: ", err) - } - for _, v := range p.NullCounts { - if err := oprot.WriteI64(ctx, int64(v)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T. (0) field write error: ", p), err) } - } - if err := oprot.WriteListEnd(ctx); err != nil { - return thrift.PrependError("error writing list end: ", err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 5:null_counts: ", p), err) } - } - return err + if p.IsSetNullCounts() { + if err := oprot.WriteFieldBegin(ctx, "null_counts", thrift.LIST, 5); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:null_counts: ", p), err) + } + if err := oprot.WriteListBegin(ctx, thrift.I64, len(p.NullCounts)); err != nil { + return thrift.PrependError("error writing list begin: ", err) + } + for _, v := range p.NullCounts { + if err := oprot.WriteI64(ctx, int64(v)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T. (0) field write error: ", p), err) + } + } + if err := oprot.WriteListEnd(ctx); err != nil { + return thrift.PrependError("error writing list end: ", err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 5:null_counts: ", p), err) + } + } + return err } func (p *ColumnIndex) Equals(other *ColumnIndex) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if len(p.NullPages) != len(other.NullPages) { return false } - for i, _tgt := range p.NullPages { - _src20 := other.NullPages[i] - if _tgt != _src20 { return false } - } - if len(p.MinValues) != len(other.MinValues) { return false } - for i, _tgt := range p.MinValues { - _src21 := other.MinValues[i] - if bytes.Compare(_tgt, _src21) != 0 { return false } - } - if len(p.MaxValues) != len(other.MaxValues) { return false } - for i, _tgt := range p.MaxValues { - _src22 := other.MaxValues[i] - if bytes.Compare(_tgt, _src22) != 0 { return false } - } - if p.BoundaryOrder != other.BoundaryOrder { return false } - if len(p.NullCounts) != len(other.NullCounts) { return false } - for i, _tgt := range p.NullCounts { - _src23 := other.NullCounts[i] - if _tgt != _src23 { return false } - } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if len(p.NullPages) != len(other.NullPages) { + return false + } + for i, _tgt := range p.NullPages { + _src20 := other.NullPages[i] + if _tgt != _src20 { + return false + } + } + if len(p.MinValues) != len(other.MinValues) { + return false + } + for i, _tgt := range p.MinValues { + _src21 := other.MinValues[i] + if bytes.Compare(_tgt, _src21) != 0 { + return false + } + } + if len(p.MaxValues) != len(other.MaxValues) { + return false + } + for i, _tgt := range p.MaxValues { + _src22 := other.MaxValues[i] + if bytes.Compare(_tgt, _src22) != 0 { + return false + } + } + if p.BoundaryOrder != other.BoundaryOrder { + return false + } + if len(p.NullCounts) != len(other.NullCounts) { + return false + } + for i, _tgt := range p.NullCounts { + _src23 := other.NullCounts[i] + if _tgt != _src23 { + return false + } + } + return true } func (p *ColumnIndex) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("ColumnIndex(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("ColumnIndex(%+v)", *p) } func (p *ColumnIndex) Validate() error { - return nil + return nil } + // Attributes: -// - AadPrefix: AAD prefix * -// - AadFileUnique: Unique file identifier part of AAD suffix * -// - SupplyAadPrefix: In files encrypted with AAD prefix without storing it, +// - AadPrefix: AAD prefix * +// - AadFileUnique: Unique file identifier part of AAD suffix * +// - SupplyAadPrefix: In files encrypted with AAD prefix without storing it, +// // readers must supply the prefix * type AesGcmV1 struct { - AadPrefix []byte `thrift:"aad_prefix,1" db:"aad_prefix" json:"aad_prefix,omitempty"` - AadFileUnique []byte `thrift:"aad_file_unique,2" db:"aad_file_unique" json:"aad_file_unique,omitempty"` - SupplyAadPrefix *bool `thrift:"supply_aad_prefix,3" db:"supply_aad_prefix" json:"supply_aad_prefix,omitempty"` + AadPrefix []byte `thrift:"aad_prefix,1" db:"aad_prefix" json:"aad_prefix,omitempty"` + AadFileUnique []byte `thrift:"aad_file_unique,2" db:"aad_file_unique" json:"aad_file_unique,omitempty"` + SupplyAadPrefix *bool `thrift:"supply_aad_prefix,3" db:"supply_aad_prefix" json:"supply_aad_prefix,omitempty"` } func NewAesGcmV1() *AesGcmV1 { - return &AesGcmV1{} + return &AesGcmV1{} } var AesGcmV1_AadPrefix_DEFAULT []byte func (p *AesGcmV1) GetAadPrefix() []byte { - return p.AadPrefix + return p.AadPrefix } + var AesGcmV1_AadFileUnique_DEFAULT []byte func (p *AesGcmV1) GetAadFileUnique() []byte { - return p.AadFileUnique + return p.AadFileUnique } + var AesGcmV1_SupplyAadPrefix_DEFAULT bool + func (p *AesGcmV1) GetSupplyAadPrefix() bool { - if !p.IsSetSupplyAadPrefix() { - return AesGcmV1_SupplyAadPrefix_DEFAULT - } -return *p.SupplyAadPrefix + if !p.IsSetSupplyAadPrefix() { + return AesGcmV1_SupplyAadPrefix_DEFAULT + } + return *p.SupplyAadPrefix } func (p *AesGcmV1) IsSetAadPrefix() bool { - return p.AadPrefix != nil + return p.AadPrefix != nil } func (p *AesGcmV1) IsSetAadFileUnique() bool { - return p.AadFileUnique != nil + return p.AadFileUnique != nil } func (p *AesGcmV1) IsSetSupplyAadPrefix() bool { - return p.SupplyAadPrefix != nil + return p.SupplyAadPrefix != nil } func (p *AesGcmV1) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.STRING { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.STRING { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 3: - if fieldTypeId == thrift.BOOL { - if err := p.ReadField3(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil -} - -func (p *AesGcmV1) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadBinary(ctx); err != nil { - return thrift.PrependError("error reading field 1: ", err) -} else { - p.AadPrefix = v -} - return nil -} - -func (p *AesGcmV1) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadBinary(ctx); err != nil { - return thrift.PrependError("error reading field 2: ", err) -} else { - p.AadFileUnique = v -} - return nil -} - -func (p *AesGcmV1) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadBool(ctx); err != nil { - return thrift.PrependError("error reading field 3: ", err) -} else { - p.SupplyAadPrefix = &v -} - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.STRING { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.STRING { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 3: + if fieldTypeId == thrift.BOOL { + if err := p.ReadField3(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil +} + +func (p *AesGcmV1) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadBinary(ctx); err != nil { + return thrift.PrependError("error reading field 1: ", err) + } else { + p.AadPrefix = v + } + return nil +} + +func (p *AesGcmV1) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadBinary(ctx); err != nil { + return thrift.PrependError("error reading field 2: ", err) + } else { + p.AadFileUnique = v + } + return nil +} + +func (p *AesGcmV1) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadBool(ctx); err != nil { + return thrift.PrependError("error reading field 3: ", err) + } else { + p.SupplyAadPrefix = &v + } + return nil } func (p *AesGcmV1) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "AesGcmV1"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - if err := p.writeField3(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "AesGcmV1"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + if err := p.writeField3(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *AesGcmV1) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetAadPrefix() { - if err := oprot.WriteFieldBegin(ctx, "aad_prefix", thrift.STRING, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:aad_prefix: ", p), err) } - if err := oprot.WriteBinary(ctx, p.AadPrefix); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.aad_prefix (1) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:aad_prefix: ", p), err) } - } - return err + if p.IsSetAadPrefix() { + if err := oprot.WriteFieldBegin(ctx, "aad_prefix", thrift.STRING, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:aad_prefix: ", p), err) + } + if err := oprot.WriteBinary(ctx, p.AadPrefix); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.aad_prefix (1) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:aad_prefix: ", p), err) + } + } + return err } func (p *AesGcmV1) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetAadFileUnique() { - if err := oprot.WriteFieldBegin(ctx, "aad_file_unique", thrift.STRING, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:aad_file_unique: ", p), err) } - if err := oprot.WriteBinary(ctx, p.AadFileUnique); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.aad_file_unique (2) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:aad_file_unique: ", p), err) } - } - return err + if p.IsSetAadFileUnique() { + if err := oprot.WriteFieldBegin(ctx, "aad_file_unique", thrift.STRING, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:aad_file_unique: ", p), err) + } + if err := oprot.WriteBinary(ctx, p.AadFileUnique); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.aad_file_unique (2) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:aad_file_unique: ", p), err) + } + } + return err } func (p *AesGcmV1) writeField3(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetSupplyAadPrefix() { - if err := oprot.WriteFieldBegin(ctx, "supply_aad_prefix", thrift.BOOL, 3); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:supply_aad_prefix: ", p), err) } - if err := oprot.WriteBool(ctx, bool(*p.SupplyAadPrefix)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.supply_aad_prefix (3) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 3:supply_aad_prefix: ", p), err) } - } - return err + if p.IsSetSupplyAadPrefix() { + if err := oprot.WriteFieldBegin(ctx, "supply_aad_prefix", thrift.BOOL, 3); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:supply_aad_prefix: ", p), err) + } + if err := oprot.WriteBool(ctx, bool(*p.SupplyAadPrefix)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.supply_aad_prefix (3) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 3:supply_aad_prefix: ", p), err) + } + } + return err } func (p *AesGcmV1) Equals(other *AesGcmV1) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if bytes.Compare(p.AadPrefix, other.AadPrefix) != 0 { return false } - if bytes.Compare(p.AadFileUnique, other.AadFileUnique) != 0 { return false } - if p.SupplyAadPrefix != other.SupplyAadPrefix { - if p.SupplyAadPrefix == nil || other.SupplyAadPrefix == nil { - return false - } - if (*p.SupplyAadPrefix) != (*other.SupplyAadPrefix) { return false } - } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if bytes.Compare(p.AadPrefix, other.AadPrefix) != 0 { + return false + } + if bytes.Compare(p.AadFileUnique, other.AadFileUnique) != 0 { + return false + } + if p.SupplyAadPrefix != other.SupplyAadPrefix { + if p.SupplyAadPrefix == nil || other.SupplyAadPrefix == nil { + return false + } + if (*p.SupplyAadPrefix) != (*other.SupplyAadPrefix) { + return false + } + } + return true } func (p *AesGcmV1) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("AesGcmV1(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("AesGcmV1(%+v)", *p) } func (p *AesGcmV1) Validate() error { - return nil + return nil } + // Attributes: -// - AadPrefix: AAD prefix * -// - AadFileUnique: Unique file identifier part of AAD suffix * -// - SupplyAadPrefix: In files encrypted with AAD prefix without storing it, +// - AadPrefix: AAD prefix * +// - AadFileUnique: Unique file identifier part of AAD suffix * +// - SupplyAadPrefix: In files encrypted with AAD prefix without storing it, +// // readers must supply the prefix * type AesGcmCtrV1 struct { - AadPrefix []byte `thrift:"aad_prefix,1" db:"aad_prefix" json:"aad_prefix,omitempty"` - AadFileUnique []byte `thrift:"aad_file_unique,2" db:"aad_file_unique" json:"aad_file_unique,omitempty"` - SupplyAadPrefix *bool `thrift:"supply_aad_prefix,3" db:"supply_aad_prefix" json:"supply_aad_prefix,omitempty"` + AadPrefix []byte `thrift:"aad_prefix,1" db:"aad_prefix" json:"aad_prefix,omitempty"` + AadFileUnique []byte `thrift:"aad_file_unique,2" db:"aad_file_unique" json:"aad_file_unique,omitempty"` + SupplyAadPrefix *bool `thrift:"supply_aad_prefix,3" db:"supply_aad_prefix" json:"supply_aad_prefix,omitempty"` } func NewAesGcmCtrV1() *AesGcmCtrV1 { - return &AesGcmCtrV1{} + return &AesGcmCtrV1{} } var AesGcmCtrV1_AadPrefix_DEFAULT []byte func (p *AesGcmCtrV1) GetAadPrefix() []byte { - return p.AadPrefix + return p.AadPrefix } + var AesGcmCtrV1_AadFileUnique_DEFAULT []byte func (p *AesGcmCtrV1) GetAadFileUnique() []byte { - return p.AadFileUnique + return p.AadFileUnique } + var AesGcmCtrV1_SupplyAadPrefix_DEFAULT bool + func (p *AesGcmCtrV1) GetSupplyAadPrefix() bool { - if !p.IsSetSupplyAadPrefix() { - return AesGcmCtrV1_SupplyAadPrefix_DEFAULT - } -return *p.SupplyAadPrefix + if !p.IsSetSupplyAadPrefix() { + return AesGcmCtrV1_SupplyAadPrefix_DEFAULT + } + return *p.SupplyAadPrefix } func (p *AesGcmCtrV1) IsSetAadPrefix() bool { - return p.AadPrefix != nil + return p.AadPrefix != nil } func (p *AesGcmCtrV1) IsSetAadFileUnique() bool { - return p.AadFileUnique != nil + return p.AadFileUnique != nil } func (p *AesGcmCtrV1) IsSetSupplyAadPrefix() bool { - return p.SupplyAadPrefix != nil + return p.SupplyAadPrefix != nil } func (p *AesGcmCtrV1) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.STRING { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.STRING { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 3: - if fieldTypeId == thrift.BOOL { - if err := p.ReadField3(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil -} - -func (p *AesGcmCtrV1) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadBinary(ctx); err != nil { - return thrift.PrependError("error reading field 1: ", err) -} else { - p.AadPrefix = v -} - return nil -} - -func (p *AesGcmCtrV1) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadBinary(ctx); err != nil { - return thrift.PrependError("error reading field 2: ", err) -} else { - p.AadFileUnique = v -} - return nil -} - -func (p *AesGcmCtrV1) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadBool(ctx); err != nil { - return thrift.PrependError("error reading field 3: ", err) -} else { - p.SupplyAadPrefix = &v -} - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.STRING { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.STRING { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 3: + if fieldTypeId == thrift.BOOL { + if err := p.ReadField3(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil +} + +func (p *AesGcmCtrV1) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadBinary(ctx); err != nil { + return thrift.PrependError("error reading field 1: ", err) + } else { + p.AadPrefix = v + } + return nil +} + +func (p *AesGcmCtrV1) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadBinary(ctx); err != nil { + return thrift.PrependError("error reading field 2: ", err) + } else { + p.AadFileUnique = v + } + return nil +} + +func (p *AesGcmCtrV1) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadBool(ctx); err != nil { + return thrift.PrependError("error reading field 3: ", err) + } else { + p.SupplyAadPrefix = &v + } + return nil } func (p *AesGcmCtrV1) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "AesGcmCtrV1"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - if err := p.writeField3(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "AesGcmCtrV1"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + if err := p.writeField3(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *AesGcmCtrV1) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetAadPrefix() { - if err := oprot.WriteFieldBegin(ctx, "aad_prefix", thrift.STRING, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:aad_prefix: ", p), err) } - if err := oprot.WriteBinary(ctx, p.AadPrefix); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.aad_prefix (1) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:aad_prefix: ", p), err) } - } - return err + if p.IsSetAadPrefix() { + if err := oprot.WriteFieldBegin(ctx, "aad_prefix", thrift.STRING, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:aad_prefix: ", p), err) + } + if err := oprot.WriteBinary(ctx, p.AadPrefix); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.aad_prefix (1) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:aad_prefix: ", p), err) + } + } + return err } func (p *AesGcmCtrV1) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetAadFileUnique() { - if err := oprot.WriteFieldBegin(ctx, "aad_file_unique", thrift.STRING, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:aad_file_unique: ", p), err) } - if err := oprot.WriteBinary(ctx, p.AadFileUnique); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.aad_file_unique (2) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:aad_file_unique: ", p), err) } - } - return err + if p.IsSetAadFileUnique() { + if err := oprot.WriteFieldBegin(ctx, "aad_file_unique", thrift.STRING, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:aad_file_unique: ", p), err) + } + if err := oprot.WriteBinary(ctx, p.AadFileUnique); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.aad_file_unique (2) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:aad_file_unique: ", p), err) + } + } + return err } func (p *AesGcmCtrV1) writeField3(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetSupplyAadPrefix() { - if err := oprot.WriteFieldBegin(ctx, "supply_aad_prefix", thrift.BOOL, 3); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:supply_aad_prefix: ", p), err) } - if err := oprot.WriteBool(ctx, bool(*p.SupplyAadPrefix)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.supply_aad_prefix (3) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 3:supply_aad_prefix: ", p), err) } - } - return err + if p.IsSetSupplyAadPrefix() { + if err := oprot.WriteFieldBegin(ctx, "supply_aad_prefix", thrift.BOOL, 3); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:supply_aad_prefix: ", p), err) + } + if err := oprot.WriteBool(ctx, bool(*p.SupplyAadPrefix)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.supply_aad_prefix (3) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 3:supply_aad_prefix: ", p), err) + } + } + return err } func (p *AesGcmCtrV1) Equals(other *AesGcmCtrV1) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if bytes.Compare(p.AadPrefix, other.AadPrefix) != 0 { return false } - if bytes.Compare(p.AadFileUnique, other.AadFileUnique) != 0 { return false } - if p.SupplyAadPrefix != other.SupplyAadPrefix { - if p.SupplyAadPrefix == nil || other.SupplyAadPrefix == nil { - return false - } - if (*p.SupplyAadPrefix) != (*other.SupplyAadPrefix) { return false } - } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if bytes.Compare(p.AadPrefix, other.AadPrefix) != 0 { + return false + } + if bytes.Compare(p.AadFileUnique, other.AadFileUnique) != 0 { + return false + } + if p.SupplyAadPrefix != other.SupplyAadPrefix { + if p.SupplyAadPrefix == nil || other.SupplyAadPrefix == nil { + return false + } + if (*p.SupplyAadPrefix) != (*other.SupplyAadPrefix) { + return false + } + } + return true } func (p *AesGcmCtrV1) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("AesGcmCtrV1(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("AesGcmCtrV1(%+v)", *p) } func (p *AesGcmCtrV1) Validate() error { - return nil + return nil } + // Attributes: -// - AES_GCM_V1 -// - AES_GCM_CTR_V1 +// - AES_GCM_V1 +// - AES_GCM_CTR_V1 type EncryptionAlgorithm struct { - AES_GCM_V1 *AesGcmV1 `thrift:"AES_GCM_V1,1" db:"AES_GCM_V1" json:"AES_GCM_V1,omitempty"` - AES_GCM_CTR_V1 *AesGcmCtrV1 `thrift:"AES_GCM_CTR_V1,2" db:"AES_GCM_CTR_V1" json:"AES_GCM_CTR_V1,omitempty"` + AES_GCM_V1 *AesGcmV1 `thrift:"AES_GCM_V1,1" db:"AES_GCM_V1" json:"AES_GCM_V1,omitempty"` + AES_GCM_CTR_V1 *AesGcmCtrV1 `thrift:"AES_GCM_CTR_V1,2" db:"AES_GCM_CTR_V1" json:"AES_GCM_CTR_V1,omitempty"` } func NewEncryptionAlgorithm() *EncryptionAlgorithm { - return &EncryptionAlgorithm{} + return &EncryptionAlgorithm{} } var EncryptionAlgorithm_AES_GCM_V1_DEFAULT *AesGcmV1 + func (p *EncryptionAlgorithm) GetAES_GCM_V1() *AesGcmV1 { - if !p.IsSetAES_GCM_V1() { - return EncryptionAlgorithm_AES_GCM_V1_DEFAULT - } -return p.AES_GCM_V1 + if !p.IsSetAES_GCM_V1() { + return EncryptionAlgorithm_AES_GCM_V1_DEFAULT + } + return p.AES_GCM_V1 } + var EncryptionAlgorithm_AES_GCM_CTR_V1_DEFAULT *AesGcmCtrV1 + func (p *EncryptionAlgorithm) GetAES_GCM_CTR_V1() *AesGcmCtrV1 { - if !p.IsSetAES_GCM_CTR_V1() { - return EncryptionAlgorithm_AES_GCM_CTR_V1_DEFAULT - } -return p.AES_GCM_CTR_V1 + if !p.IsSetAES_GCM_CTR_V1() { + return EncryptionAlgorithm_AES_GCM_CTR_V1_DEFAULT + } + return p.AES_GCM_CTR_V1 } func (p *EncryptionAlgorithm) CountSetFieldsEncryptionAlgorithm() int { - count := 0 - if (p.IsSetAES_GCM_V1()) { - count++ - } - if (p.IsSetAES_GCM_CTR_V1()) { - count++ - } - return count + count := 0 + if p.IsSetAES_GCM_V1() { + count++ + } + if p.IsSetAES_GCM_CTR_V1() { + count++ + } + return count } func (p *EncryptionAlgorithm) IsSetAES_GCM_V1() bool { - return p.AES_GCM_V1 != nil + return p.AES_GCM_V1 != nil } func (p *EncryptionAlgorithm) IsSetAES_GCM_CTR_V1() bool { - return p.AES_GCM_CTR_V1 != nil + return p.AES_GCM_CTR_V1 != nil } func (p *EncryptionAlgorithm) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - return nil -} - -func (p *EncryptionAlgorithm) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - p.AES_GCM_V1 = &AesGcmV1{} - if err := p.AES_GCM_V1.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.AES_GCM_V1), err) - } - return nil -} - -func (p *EncryptionAlgorithm) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - p.AES_GCM_CTR_V1 = &AesGcmCtrV1{} - if err := p.AES_GCM_CTR_V1.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.AES_GCM_CTR_V1), err) - } - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil +} + +func (p *EncryptionAlgorithm) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + p.AES_GCM_V1 = &AesGcmV1{} + if err := p.AES_GCM_V1.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.AES_GCM_V1), err) + } + return nil +} + +func (p *EncryptionAlgorithm) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + p.AES_GCM_CTR_V1 = &AesGcmCtrV1{} + if err := p.AES_GCM_CTR_V1.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.AES_GCM_CTR_V1), err) + } + return nil } func (p *EncryptionAlgorithm) Write(ctx context.Context, oprot thrift.TProtocol) error { - if c := p.CountSetFieldsEncryptionAlgorithm(); c != 1 { - return fmt.Errorf("%T write union: exactly one field must be set (%d set)", p, c) - } - if err := oprot.WriteStructBegin(ctx, "EncryptionAlgorithm"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if c := p.CountSetFieldsEncryptionAlgorithm(); c != 1 { + return fmt.Errorf("%T write union: exactly one field must be set (%d set)", p, c) + } + if err := oprot.WriteStructBegin(ctx, "EncryptionAlgorithm"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *EncryptionAlgorithm) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetAES_GCM_V1() { - if err := oprot.WriteFieldBegin(ctx, "AES_GCM_V1", thrift.STRUCT, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:AES_GCM_V1: ", p), err) } - if err := p.AES_GCM_V1.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.AES_GCM_V1), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:AES_GCM_V1: ", p), err) } - } - return err + if p.IsSetAES_GCM_V1() { + if err := oprot.WriteFieldBegin(ctx, "AES_GCM_V1", thrift.STRUCT, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:AES_GCM_V1: ", p), err) + } + if err := p.AES_GCM_V1.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.AES_GCM_V1), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:AES_GCM_V1: ", p), err) + } + } + return err } func (p *EncryptionAlgorithm) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetAES_GCM_CTR_V1() { - if err := oprot.WriteFieldBegin(ctx, "AES_GCM_CTR_V1", thrift.STRUCT, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:AES_GCM_CTR_V1: ", p), err) } - if err := p.AES_GCM_CTR_V1.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.AES_GCM_CTR_V1), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:AES_GCM_CTR_V1: ", p), err) } - } - return err + if p.IsSetAES_GCM_CTR_V1() { + if err := oprot.WriteFieldBegin(ctx, "AES_GCM_CTR_V1", thrift.STRUCT, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:AES_GCM_CTR_V1: ", p), err) + } + if err := p.AES_GCM_CTR_V1.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.AES_GCM_CTR_V1), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:AES_GCM_CTR_V1: ", p), err) + } + } + return err } func (p *EncryptionAlgorithm) Equals(other *EncryptionAlgorithm) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if !p.AES_GCM_V1.Equals(other.AES_GCM_V1) { return false } - if !p.AES_GCM_CTR_V1.Equals(other.AES_GCM_CTR_V1) { return false } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if !p.AES_GCM_V1.Equals(other.AES_GCM_V1) { + return false + } + if !p.AES_GCM_CTR_V1.Equals(other.AES_GCM_CTR_V1) { + return false + } + return true } func (p *EncryptionAlgorithm) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("EncryptionAlgorithm(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("EncryptionAlgorithm(%+v)", *p) } func (p *EncryptionAlgorithm) Validate() error { - return nil + return nil } + // Description for file metadata -// +// // Attributes: -// - Version: Version of this file * -// - Schema: Parquet schema for this file. This schema contains metadata for all the columns. +// - Version: Version of this file * +// - Schema: Parquet schema for this file. This schema contains metadata for all the columns. +// // The schema is represented as a tree with a single root. The nodes of the tree // are flattened to a list by doing a depth-first traversal. // The column metadata contains the path in the schema for that column which can be // used to map columns to nodes in the schema. // The first element is the root * -// - NumRows: Number of rows in this file * -// - RowGroups: Row groups in this file * -// - KeyValueMetadata: Optional key/value metadata * -// - CreatedBy: String for application that wrote this file. This should be in the format +// - NumRows: Number of rows in this file * +// - RowGroups: Row groups in this file * +// - KeyValueMetadata: Optional key/value metadata * +// - CreatedBy: String for application that wrote this file. This should be in the format +// // version (build ). // e.g. impala version 1.0 (build 6cf94d29b2b7115df4de2c06e2ab4326d721eb55) -// -// - ColumnOrders: Sort order used for the min_value and max_value fields in the Statistics +// +// - ColumnOrders: Sort order used for the min_value and max_value fields in the Statistics +// // objects and the min_values and max_values fields in the ColumnIndex // objects of each column in this file. Sort orders are listed in the order // matching the columns in the schema. The indexes are not necessary the same // though, because only leaf nodes of the schema are represented in the list // of sort orders. -// +// // Without column_orders, the meaning of the min_value and max_value fields // in the Statistics object and the ColumnIndex object is undefined. To ensure // well-defined behaviour, if these fields are written to a Parquet file, // column_orders must be written as well. -// +// // The obsolete min and max fields in the Statistics object are always sorted // by signed comparison regardless of column_orders. -// - EncryptionAlgorithm: Encryption algorithm. This field is set only in encrypted files +// - EncryptionAlgorithm: Encryption algorithm. This field is set only in encrypted files +// // with plaintext footer. Files with encrypted footer store algorithm id // in FileCryptoMetaData structure. -// - FooterSigningKeyMetadata: Retrieval metadata of key used for signing the footer. +// - FooterSigningKeyMetadata: Retrieval metadata of key used for signing the footer. +// // Used only in encrypted files with plaintext footer. type FileMetaData struct { - Version int32 `thrift:"version,1,required" db:"version" json:"version"` - Schema []*SchemaElement `thrift:"schema,2,required" db:"schema" json:"schema"` - NumRows int64 `thrift:"num_rows,3,required" db:"num_rows" json:"num_rows"` - RowGroups []*RowGroup `thrift:"row_groups,4,required" db:"row_groups" json:"row_groups"` - KeyValueMetadata []*KeyValue `thrift:"key_value_metadata,5" db:"key_value_metadata" json:"key_value_metadata,omitempty"` - CreatedBy *string `thrift:"created_by,6" db:"created_by" json:"created_by,omitempty"` - ColumnOrders []*ColumnOrder `thrift:"column_orders,7" db:"column_orders" json:"column_orders,omitempty"` - EncryptionAlgorithm *EncryptionAlgorithm `thrift:"encryption_algorithm,8" db:"encryption_algorithm" json:"encryption_algorithm,omitempty"` - FooterSigningKeyMetadata []byte `thrift:"footer_signing_key_metadata,9" db:"footer_signing_key_metadata" json:"footer_signing_key_metadata,omitempty"` + Version int32 `thrift:"version,1,required" db:"version" json:"version"` + Schema []*SchemaElement `thrift:"schema,2,required" db:"schema" json:"schema"` + NumRows int64 `thrift:"num_rows,3,required" db:"num_rows" json:"num_rows"` + RowGroups []*RowGroup `thrift:"row_groups,4,required" db:"row_groups" json:"row_groups"` + KeyValueMetadata []*KeyValue `thrift:"key_value_metadata,5" db:"key_value_metadata" json:"key_value_metadata,omitempty"` + CreatedBy *string `thrift:"created_by,6" db:"created_by" json:"created_by,omitempty"` + ColumnOrders []*ColumnOrder `thrift:"column_orders,7" db:"column_orders" json:"column_orders,omitempty"` + EncryptionAlgorithm *EncryptionAlgorithm `thrift:"encryption_algorithm,8" db:"encryption_algorithm" json:"encryption_algorithm,omitempty"` + FooterSigningKeyMetadata []byte `thrift:"footer_signing_key_metadata,9" db:"footer_signing_key_metadata" json:"footer_signing_key_metadata,omitempty"` } func NewFileMetaData() *FileMetaData { - return &FileMetaData{} + return &FileMetaData{} } - func (p *FileMetaData) GetVersion() int32 { - return p.Version + return p.Version } func (p *FileMetaData) GetSchema() []*SchemaElement { - return p.Schema + return p.Schema } func (p *FileMetaData) GetNumRows() int64 { - return p.NumRows + return p.NumRows } func (p *FileMetaData) GetRowGroups() []*RowGroup { - return p.RowGroups + return p.RowGroups } + var FileMetaData_KeyValueMetadata_DEFAULT []*KeyValue func (p *FileMetaData) GetKeyValueMetadata() []*KeyValue { - return p.KeyValueMetadata + return p.KeyValueMetadata } + var FileMetaData_CreatedBy_DEFAULT string + func (p *FileMetaData) GetCreatedBy() string { - if !p.IsSetCreatedBy() { - return FileMetaData_CreatedBy_DEFAULT - } -return *p.CreatedBy + if !p.IsSetCreatedBy() { + return FileMetaData_CreatedBy_DEFAULT + } + return *p.CreatedBy } + var FileMetaData_ColumnOrders_DEFAULT []*ColumnOrder func (p *FileMetaData) GetColumnOrders() []*ColumnOrder { - return p.ColumnOrders + return p.ColumnOrders } + var FileMetaData_EncryptionAlgorithm_DEFAULT *EncryptionAlgorithm + func (p *FileMetaData) GetEncryptionAlgorithm() *EncryptionAlgorithm { - if !p.IsSetEncryptionAlgorithm() { - return FileMetaData_EncryptionAlgorithm_DEFAULT - } -return p.EncryptionAlgorithm + if !p.IsSetEncryptionAlgorithm() { + return FileMetaData_EncryptionAlgorithm_DEFAULT + } + return p.EncryptionAlgorithm } + var FileMetaData_FooterSigningKeyMetadata_DEFAULT []byte func (p *FileMetaData) GetFooterSigningKeyMetadata() []byte { - return p.FooterSigningKeyMetadata + return p.FooterSigningKeyMetadata } func (p *FileMetaData) IsSetKeyValueMetadata() bool { - return p.KeyValueMetadata != nil + return p.KeyValueMetadata != nil } func (p *FileMetaData) IsSetCreatedBy() bool { - return p.CreatedBy != nil + return p.CreatedBy != nil } func (p *FileMetaData) IsSetColumnOrders() bool { - return p.ColumnOrders != nil + return p.ColumnOrders != nil } func (p *FileMetaData) IsSetEncryptionAlgorithm() bool { - return p.EncryptionAlgorithm != nil + return p.EncryptionAlgorithm != nil } func (p *FileMetaData) IsSetFooterSigningKeyMetadata() bool { - return p.FooterSigningKeyMetadata != nil + return p.FooterSigningKeyMetadata != nil } func (p *FileMetaData) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - var issetVersion bool = false; - var issetSchema bool = false; - var issetNumRows bool = false; - var issetRowGroups bool = false; - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.I32 { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - issetVersion = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.LIST { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - issetSchema = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 3: - if fieldTypeId == thrift.I64 { - if err := p.ReadField3(ctx, iprot); err != nil { - return err - } - issetNumRows = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 4: - if fieldTypeId == thrift.LIST { - if err := p.ReadField4(ctx, iprot); err != nil { - return err - } - issetRowGroups = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 5: - if fieldTypeId == thrift.LIST { - if err := p.ReadField5(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 6: - if fieldTypeId == thrift.STRING { - if err := p.ReadField6(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 7: - if fieldTypeId == thrift.LIST { - if err := p.ReadField7(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 8: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField8(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 9: - if fieldTypeId == thrift.STRING { - if err := p.ReadField9(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - if !issetVersion{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Version is not set")); - } - if !issetSchema{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Schema is not set")); - } - if !issetNumRows{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field NumRows is not set")); - } - if !issetRowGroups{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field RowGroups is not set")); - } - return nil -} - -func (p *FileMetaData) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI32(ctx); err != nil { - return thrift.PrependError("error reading field 1: ", err) -} else { - p.Version = v -} - return nil -} - -func (p *FileMetaData) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - _, size, err := iprot.ReadListBegin(ctx) - if err != nil { - return thrift.PrependError("error reading list begin: ", err) - } - tSlice := make([]*SchemaElement, 0, size) - p.Schema = tSlice - for i := 0; i < size; i ++ { - _elem24 := &SchemaElement{} - if err := _elem24.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem24), err) - } - p.Schema = append(p.Schema, _elem24) - } - if err := iprot.ReadListEnd(ctx); err != nil { - return thrift.PrependError("error reading list end: ", err) - } - return nil -} - -func (p *FileMetaData) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadI64(ctx); err != nil { - return thrift.PrependError("error reading field 3: ", err) -} else { - p.NumRows = v -} - return nil -} - -func (p *FileMetaData) ReadField4(ctx context.Context, iprot thrift.TProtocol) error { - _, size, err := iprot.ReadListBegin(ctx) - if err != nil { - return thrift.PrependError("error reading list begin: ", err) - } - tSlice := make([]*RowGroup, 0, size) - p.RowGroups = tSlice - for i := 0; i < size; i ++ { - _elem25 := &RowGroup{} - if err := _elem25.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem25), err) - } - p.RowGroups = append(p.RowGroups, _elem25) - } - if err := iprot.ReadListEnd(ctx); err != nil { - return thrift.PrependError("error reading list end: ", err) - } - return nil -} - -func (p *FileMetaData) ReadField5(ctx context.Context, iprot thrift.TProtocol) error { - _, size, err := iprot.ReadListBegin(ctx) - if err != nil { - return thrift.PrependError("error reading list begin: ", err) - } - tSlice := make([]*KeyValue, 0, size) - p.KeyValueMetadata = tSlice - for i := 0; i < size; i ++ { - _elem26 := &KeyValue{} - if err := _elem26.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem26), err) - } - p.KeyValueMetadata = append(p.KeyValueMetadata, _elem26) - } - if err := iprot.ReadListEnd(ctx); err != nil { - return thrift.PrependError("error reading list end: ", err) - } - return nil -} - -func (p *FileMetaData) ReadField6(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadString(ctx); err != nil { - return thrift.PrependError("error reading field 6: ", err) -} else { - p.CreatedBy = &v -} - return nil -} - -func (p *FileMetaData) ReadField7(ctx context.Context, iprot thrift.TProtocol) error { - _, size, err := iprot.ReadListBegin(ctx) - if err != nil { - return thrift.PrependError("error reading list begin: ", err) - } - tSlice := make([]*ColumnOrder, 0, size) - p.ColumnOrders = tSlice - for i := 0; i < size; i ++ { - _elem27 := &ColumnOrder{} - if err := _elem27.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem27), err) - } - p.ColumnOrders = append(p.ColumnOrders, _elem27) - } - if err := iprot.ReadListEnd(ctx); err != nil { - return thrift.PrependError("error reading list end: ", err) - } - return nil -} - -func (p *FileMetaData) ReadField8(ctx context.Context, iprot thrift.TProtocol) error { - p.EncryptionAlgorithm = &EncryptionAlgorithm{} - if err := p.EncryptionAlgorithm.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.EncryptionAlgorithm), err) - } - return nil -} - -func (p *FileMetaData) ReadField9(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadBinary(ctx); err != nil { - return thrift.PrependError("error reading field 9: ", err) -} else { - p.FooterSigningKeyMetadata = v -} - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + var issetVersion bool = false + var issetSchema bool = false + var issetNumRows bool = false + var issetRowGroups bool = false + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.I32 { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + issetVersion = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.LIST { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + issetSchema = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 3: + if fieldTypeId == thrift.I64 { + if err := p.ReadField3(ctx, iprot); err != nil { + return err + } + issetNumRows = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 4: + if fieldTypeId == thrift.LIST { + if err := p.ReadField4(ctx, iprot); err != nil { + return err + } + issetRowGroups = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 5: + if fieldTypeId == thrift.LIST { + if err := p.ReadField5(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 6: + if fieldTypeId == thrift.STRING { + if err := p.ReadField6(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 7: + if fieldTypeId == thrift.LIST { + if err := p.ReadField7(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 8: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField8(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 9: + if fieldTypeId == thrift.STRING { + if err := p.ReadField9(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + if !issetVersion { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Version is not set")) + } + if !issetSchema { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field Schema is not set")) + } + if !issetNumRows { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field NumRows is not set")) + } + if !issetRowGroups { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field RowGroups is not set")) + } + return nil +} + +func (p *FileMetaData) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 1: ", err) + } else { + p.Version = v + } + return nil +} + +func (p *FileMetaData) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + _, size, err := iprot.ReadListBegin(ctx) + if err != nil { + return thrift.PrependError("error reading list begin: ", err) + } + tSlice := make([]*SchemaElement, 0, size) + p.Schema = tSlice + for i := 0; i < size; i++ { + _elem24 := &SchemaElement{} + if err := _elem24.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem24), err) + } + p.Schema = append(p.Schema, _elem24) + } + if err := iprot.ReadListEnd(ctx); err != nil { + return thrift.PrependError("error reading list end: ", err) + } + return nil +} + +func (p *FileMetaData) ReadField3(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI64(ctx); err != nil { + return thrift.PrependError("error reading field 3: ", err) + } else { + p.NumRows = v + } + return nil +} + +func (p *FileMetaData) ReadField4(ctx context.Context, iprot thrift.TProtocol) error { + _, size, err := iprot.ReadListBegin(ctx) + if err != nil { + return thrift.PrependError("error reading list begin: ", err) + } + tSlice := make([]*RowGroup, 0, size) + p.RowGroups = tSlice + for i := 0; i < size; i++ { + _elem25 := &RowGroup{} + if err := _elem25.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem25), err) + } + p.RowGroups = append(p.RowGroups, _elem25) + } + if err := iprot.ReadListEnd(ctx); err != nil { + return thrift.PrependError("error reading list end: ", err) + } + return nil +} + +func (p *FileMetaData) ReadField5(ctx context.Context, iprot thrift.TProtocol) error { + _, size, err := iprot.ReadListBegin(ctx) + if err != nil { + return thrift.PrependError("error reading list begin: ", err) + } + tSlice := make([]*KeyValue, 0, size) + p.KeyValueMetadata = tSlice + for i := 0; i < size; i++ { + _elem26 := &KeyValue{} + if err := _elem26.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem26), err) + } + p.KeyValueMetadata = append(p.KeyValueMetadata, _elem26) + } + if err := iprot.ReadListEnd(ctx); err != nil { + return thrift.PrependError("error reading list end: ", err) + } + return nil +} + +func (p *FileMetaData) ReadField6(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadString(ctx); err != nil { + return thrift.PrependError("error reading field 6: ", err) + } else { + p.CreatedBy = &v + } + return nil +} + +func (p *FileMetaData) ReadField7(ctx context.Context, iprot thrift.TProtocol) error { + _, size, err := iprot.ReadListBegin(ctx) + if err != nil { + return thrift.PrependError("error reading list begin: ", err) + } + tSlice := make([]*ColumnOrder, 0, size) + p.ColumnOrders = tSlice + for i := 0; i < size; i++ { + _elem27 := &ColumnOrder{} + if err := _elem27.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", _elem27), err) + } + p.ColumnOrders = append(p.ColumnOrders, _elem27) + } + if err := iprot.ReadListEnd(ctx); err != nil { + return thrift.PrependError("error reading list end: ", err) + } + return nil +} + +func (p *FileMetaData) ReadField8(ctx context.Context, iprot thrift.TProtocol) error { + p.EncryptionAlgorithm = &EncryptionAlgorithm{} + if err := p.EncryptionAlgorithm.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.EncryptionAlgorithm), err) + } + return nil +} + +func (p *FileMetaData) ReadField9(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadBinary(ctx); err != nil { + return thrift.PrependError("error reading field 9: ", err) + } else { + p.FooterSigningKeyMetadata = v + } + return nil } func (p *FileMetaData) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "FileMetaData"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - if err := p.writeField3(ctx, oprot); err != nil { return err } - if err := p.writeField4(ctx, oprot); err != nil { return err } - if err := p.writeField5(ctx, oprot); err != nil { return err } - if err := p.writeField6(ctx, oprot); err != nil { return err } - if err := p.writeField7(ctx, oprot); err != nil { return err } - if err := p.writeField8(ctx, oprot); err != nil { return err } - if err := p.writeField9(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "FileMetaData"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + if err := p.writeField3(ctx, oprot); err != nil { + return err + } + if err := p.writeField4(ctx, oprot); err != nil { + return err + } + if err := p.writeField5(ctx, oprot); err != nil { + return err + } + if err := p.writeField6(ctx, oprot); err != nil { + return err + } + if err := p.writeField7(ctx, oprot); err != nil { + return err + } + if err := p.writeField8(ctx, oprot); err != nil { + return err + } + if err := p.writeField9(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *FileMetaData) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "version", thrift.I32, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:version: ", p), err) } - if err := oprot.WriteI32(ctx, int32(p.Version)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.version (1) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:version: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "version", thrift.I32, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:version: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.Version)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.version (1) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:version: ", p), err) + } + return err } func (p *FileMetaData) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "schema", thrift.LIST, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:schema: ", p), err) } - if err := oprot.WriteListBegin(ctx, thrift.STRUCT, len(p.Schema)); err != nil { - return thrift.PrependError("error writing list begin: ", err) - } - for _, v := range p.Schema { - if err := v.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", v), err) - } - } - if err := oprot.WriteListEnd(ctx); err != nil { - return thrift.PrependError("error writing list end: ", err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:schema: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "schema", thrift.LIST, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:schema: ", p), err) + } + if err := oprot.WriteListBegin(ctx, thrift.STRUCT, len(p.Schema)); err != nil { + return thrift.PrependError("error writing list begin: ", err) + } + for _, v := range p.Schema { + if err := v.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", v), err) + } + } + if err := oprot.WriteListEnd(ctx); err != nil { + return thrift.PrependError("error writing list end: ", err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:schema: ", p), err) + } + return err } func (p *FileMetaData) writeField3(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "num_rows", thrift.I64, 3); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:num_rows: ", p), err) } - if err := oprot.WriteI64(ctx, int64(p.NumRows)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.num_rows (3) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 3:num_rows: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "num_rows", thrift.I64, 3); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 3:num_rows: ", p), err) + } + if err := oprot.WriteI64(ctx, int64(p.NumRows)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.num_rows (3) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 3:num_rows: ", p), err) + } + return err } func (p *FileMetaData) writeField4(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "row_groups", thrift.LIST, 4); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:row_groups: ", p), err) } - if err := oprot.WriteListBegin(ctx, thrift.STRUCT, len(p.RowGroups)); err != nil { - return thrift.PrependError("error writing list begin: ", err) - } - for _, v := range p.RowGroups { - if err := v.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", v), err) - } - } - if err := oprot.WriteListEnd(ctx); err != nil { - return thrift.PrependError("error writing list end: ", err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 4:row_groups: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "row_groups", thrift.LIST, 4); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 4:row_groups: ", p), err) + } + if err := oprot.WriteListBegin(ctx, thrift.STRUCT, len(p.RowGroups)); err != nil { + return thrift.PrependError("error writing list begin: ", err) + } + for _, v := range p.RowGroups { + if err := v.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", v), err) + } + } + if err := oprot.WriteListEnd(ctx); err != nil { + return thrift.PrependError("error writing list end: ", err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 4:row_groups: ", p), err) + } + return err } func (p *FileMetaData) writeField5(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetKeyValueMetadata() { - if err := oprot.WriteFieldBegin(ctx, "key_value_metadata", thrift.LIST, 5); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:key_value_metadata: ", p), err) } - if err := oprot.WriteListBegin(ctx, thrift.STRUCT, len(p.KeyValueMetadata)); err != nil { - return thrift.PrependError("error writing list begin: ", err) - } - for _, v := range p.KeyValueMetadata { - if err := v.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", v), err) - } - } - if err := oprot.WriteListEnd(ctx); err != nil { - return thrift.PrependError("error writing list end: ", err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 5:key_value_metadata: ", p), err) } - } - return err + if p.IsSetKeyValueMetadata() { + if err := oprot.WriteFieldBegin(ctx, "key_value_metadata", thrift.LIST, 5); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 5:key_value_metadata: ", p), err) + } + if err := oprot.WriteListBegin(ctx, thrift.STRUCT, len(p.KeyValueMetadata)); err != nil { + return thrift.PrependError("error writing list begin: ", err) + } + for _, v := range p.KeyValueMetadata { + if err := v.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", v), err) + } + } + if err := oprot.WriteListEnd(ctx); err != nil { + return thrift.PrependError("error writing list end: ", err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 5:key_value_metadata: ", p), err) + } + } + return err } func (p *FileMetaData) writeField6(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetCreatedBy() { - if err := oprot.WriteFieldBegin(ctx, "created_by", thrift.STRING, 6); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 6:created_by: ", p), err) } - if err := oprot.WriteString(ctx, string(*p.CreatedBy)); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.created_by (6) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 6:created_by: ", p), err) } - } - return err + if p.IsSetCreatedBy() { + if err := oprot.WriteFieldBegin(ctx, "created_by", thrift.STRING, 6); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 6:created_by: ", p), err) + } + if err := oprot.WriteString(ctx, string(*p.CreatedBy)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.created_by (6) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 6:created_by: ", p), err) + } + } + return err } func (p *FileMetaData) writeField7(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetColumnOrders() { - if err := oprot.WriteFieldBegin(ctx, "column_orders", thrift.LIST, 7); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 7:column_orders: ", p), err) } - if err := oprot.WriteListBegin(ctx, thrift.STRUCT, len(p.ColumnOrders)); err != nil { - return thrift.PrependError("error writing list begin: ", err) - } - for _, v := range p.ColumnOrders { - if err := v.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", v), err) - } - } - if err := oprot.WriteListEnd(ctx); err != nil { - return thrift.PrependError("error writing list end: ", err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 7:column_orders: ", p), err) } - } - return err + if p.IsSetColumnOrders() { + if err := oprot.WriteFieldBegin(ctx, "column_orders", thrift.LIST, 7); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 7:column_orders: ", p), err) + } + if err := oprot.WriteListBegin(ctx, thrift.STRUCT, len(p.ColumnOrders)); err != nil { + return thrift.PrependError("error writing list begin: ", err) + } + for _, v := range p.ColumnOrders { + if err := v.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", v), err) + } + } + if err := oprot.WriteListEnd(ctx); err != nil { + return thrift.PrependError("error writing list end: ", err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 7:column_orders: ", p), err) + } + } + return err } func (p *FileMetaData) writeField8(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetEncryptionAlgorithm() { - if err := oprot.WriteFieldBegin(ctx, "encryption_algorithm", thrift.STRUCT, 8); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 8:encryption_algorithm: ", p), err) } - if err := p.EncryptionAlgorithm.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.EncryptionAlgorithm), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 8:encryption_algorithm: ", p), err) } - } - return err + if p.IsSetEncryptionAlgorithm() { + if err := oprot.WriteFieldBegin(ctx, "encryption_algorithm", thrift.STRUCT, 8); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 8:encryption_algorithm: ", p), err) + } + if err := p.EncryptionAlgorithm.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.EncryptionAlgorithm), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 8:encryption_algorithm: ", p), err) + } + } + return err } func (p *FileMetaData) writeField9(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetFooterSigningKeyMetadata() { - if err := oprot.WriteFieldBegin(ctx, "footer_signing_key_metadata", thrift.STRING, 9); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 9:footer_signing_key_metadata: ", p), err) } - if err := oprot.WriteBinary(ctx, p.FooterSigningKeyMetadata); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.footer_signing_key_metadata (9) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 9:footer_signing_key_metadata: ", p), err) } - } - return err + if p.IsSetFooterSigningKeyMetadata() { + if err := oprot.WriteFieldBegin(ctx, "footer_signing_key_metadata", thrift.STRING, 9); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 9:footer_signing_key_metadata: ", p), err) + } + if err := oprot.WriteBinary(ctx, p.FooterSigningKeyMetadata); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.footer_signing_key_metadata (9) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 9:footer_signing_key_metadata: ", p), err) + } + } + return err } func (p *FileMetaData) Equals(other *FileMetaData) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if p.Version != other.Version { return false } - if len(p.Schema) != len(other.Schema) { return false } - for i, _tgt := range p.Schema { - _src28 := other.Schema[i] - if !_tgt.Equals(_src28) { return false } - } - if p.NumRows != other.NumRows { return false } - if len(p.RowGroups) != len(other.RowGroups) { return false } - for i, _tgt := range p.RowGroups { - _src29 := other.RowGroups[i] - if !_tgt.Equals(_src29) { return false } - } - if len(p.KeyValueMetadata) != len(other.KeyValueMetadata) { return false } - for i, _tgt := range p.KeyValueMetadata { - _src30 := other.KeyValueMetadata[i] - if !_tgt.Equals(_src30) { return false } - } - if p.CreatedBy != other.CreatedBy { - if p.CreatedBy == nil || other.CreatedBy == nil { - return false - } - if (*p.CreatedBy) != (*other.CreatedBy) { return false } - } - if len(p.ColumnOrders) != len(other.ColumnOrders) { return false } - for i, _tgt := range p.ColumnOrders { - _src31 := other.ColumnOrders[i] - if !_tgt.Equals(_src31) { return false } - } - if !p.EncryptionAlgorithm.Equals(other.EncryptionAlgorithm) { return false } - if bytes.Compare(p.FooterSigningKeyMetadata, other.FooterSigningKeyMetadata) != 0 { return false } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if p.Version != other.Version { + return false + } + if len(p.Schema) != len(other.Schema) { + return false + } + for i, _tgt := range p.Schema { + _src28 := other.Schema[i] + if !_tgt.Equals(_src28) { + return false + } + } + if p.NumRows != other.NumRows { + return false + } + if len(p.RowGroups) != len(other.RowGroups) { + return false + } + for i, _tgt := range p.RowGroups { + _src29 := other.RowGroups[i] + if !_tgt.Equals(_src29) { + return false + } + } + if len(p.KeyValueMetadata) != len(other.KeyValueMetadata) { + return false + } + for i, _tgt := range p.KeyValueMetadata { + _src30 := other.KeyValueMetadata[i] + if !_tgt.Equals(_src30) { + return false + } + } + if p.CreatedBy != other.CreatedBy { + if p.CreatedBy == nil || other.CreatedBy == nil { + return false + } + if (*p.CreatedBy) != (*other.CreatedBy) { + return false + } + } + if len(p.ColumnOrders) != len(other.ColumnOrders) { + return false + } + for i, _tgt := range p.ColumnOrders { + _src31 := other.ColumnOrders[i] + if !_tgt.Equals(_src31) { + return false + } + } + if !p.EncryptionAlgorithm.Equals(other.EncryptionAlgorithm) { + return false + } + if bytes.Compare(p.FooterSigningKeyMetadata, other.FooterSigningKeyMetadata) != 0 { + return false + } + return true } func (p *FileMetaData) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("FileMetaData(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("FileMetaData(%+v)", *p) } func (p *FileMetaData) Validate() error { - return nil + return nil } + // Crypto metadata for files with encrypted footer * -// +// // Attributes: -// - EncryptionAlgorithm: Encryption algorithm. This field is only used for files +// - EncryptionAlgorithm: Encryption algorithm. This field is only used for files +// // with encrypted footer. Files with plaintext footer store algorithm id // inside footer (FileMetaData structure). -// - KeyMetadata: Retrieval metadata of key used for encryption of footer, +// - KeyMetadata: Retrieval metadata of key used for encryption of footer, +// // and (possibly) columns * type FileCryptoMetaData struct { - EncryptionAlgorithm *EncryptionAlgorithm `thrift:"encryption_algorithm,1,required" db:"encryption_algorithm" json:"encryption_algorithm"` - KeyMetadata []byte `thrift:"key_metadata,2" db:"key_metadata" json:"key_metadata,omitempty"` + EncryptionAlgorithm *EncryptionAlgorithm `thrift:"encryption_algorithm,1,required" db:"encryption_algorithm" json:"encryption_algorithm"` + KeyMetadata []byte `thrift:"key_metadata,2" db:"key_metadata" json:"key_metadata,omitempty"` } func NewFileCryptoMetaData() *FileCryptoMetaData { - return &FileCryptoMetaData{} + return &FileCryptoMetaData{} } var FileCryptoMetaData_EncryptionAlgorithm_DEFAULT *EncryptionAlgorithm + func (p *FileCryptoMetaData) GetEncryptionAlgorithm() *EncryptionAlgorithm { - if !p.IsSetEncryptionAlgorithm() { - return FileCryptoMetaData_EncryptionAlgorithm_DEFAULT - } -return p.EncryptionAlgorithm + if !p.IsSetEncryptionAlgorithm() { + return FileCryptoMetaData_EncryptionAlgorithm_DEFAULT + } + return p.EncryptionAlgorithm } + var FileCryptoMetaData_KeyMetadata_DEFAULT []byte func (p *FileCryptoMetaData) GetKeyMetadata() []byte { - return p.KeyMetadata + return p.KeyMetadata } func (p *FileCryptoMetaData) IsSetEncryptionAlgorithm() bool { - return p.EncryptionAlgorithm != nil + return p.EncryptionAlgorithm != nil } func (p *FileCryptoMetaData) IsSetKeyMetadata() bool { - return p.KeyMetadata != nil + return p.KeyMetadata != nil } func (p *FileCryptoMetaData) Read(ctx context.Context, iprot thrift.TProtocol) error { - if _, err := iprot.ReadStructBegin(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) - } - - var issetEncryptionAlgorithm bool = false; - - for { - _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) - if err != nil { - return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) - } - if fieldTypeId == thrift.STOP { break; } - switch fieldId { - case 1: - if fieldTypeId == thrift.STRUCT { - if err := p.ReadField1(ctx, iprot); err != nil { - return err - } - issetEncryptionAlgorithm = true - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - case 2: - if fieldTypeId == thrift.STRING { - if err := p.ReadField2(ctx, iprot); err != nil { - return err - } - } else { - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - default: - if err := iprot.Skip(ctx, fieldTypeId); err != nil { - return err - } - } - if err := iprot.ReadFieldEnd(ctx); err != nil { - return err - } - } - if err := iprot.ReadStructEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) - } - if !issetEncryptionAlgorithm{ - return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field EncryptionAlgorithm is not set")); - } - return nil -} - -func (p *FileCryptoMetaData) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { - p.EncryptionAlgorithm = &EncryptionAlgorithm{} - if err := p.EncryptionAlgorithm.Read(ctx, iprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.EncryptionAlgorithm), err) - } - return nil -} - -func (p *FileCryptoMetaData) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { - if v, err := iprot.ReadBinary(ctx); err != nil { - return thrift.PrependError("error reading field 2: ", err) -} else { - p.KeyMetadata = v -} - return nil + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + var issetEncryptionAlgorithm bool = false + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + issetEncryptionAlgorithm = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 2: + if fieldTypeId == thrift.STRING { + if err := p.ReadField2(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + if !issetEncryptionAlgorithm { + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field EncryptionAlgorithm is not set")) + } + return nil +} + +func (p *FileCryptoMetaData) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + p.EncryptionAlgorithm = &EncryptionAlgorithm{} + if err := p.EncryptionAlgorithm.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.EncryptionAlgorithm), err) + } + return nil +} + +func (p *FileCryptoMetaData) ReadField2(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadBinary(ctx); err != nil { + return thrift.PrependError("error reading field 2: ", err) + } else { + p.KeyMetadata = v + } + return nil } func (p *FileCryptoMetaData) Write(ctx context.Context, oprot thrift.TProtocol) error { - if err := oprot.WriteStructBegin(ctx, "FileCryptoMetaData"); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) } - if p != nil { - if err := p.writeField1(ctx, oprot); err != nil { return err } - if err := p.writeField2(ctx, oprot); err != nil { return err } - } - if err := oprot.WriteFieldStop(ctx); err != nil { - return thrift.PrependError("write field stop error: ", err) } - if err := oprot.WriteStructEnd(ctx); err != nil { - return thrift.PrependError("write struct stop error: ", err) } - return nil + if err := oprot.WriteStructBegin(ctx, "FileCryptoMetaData"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { + return err + } + if err := p.writeField2(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil } func (p *FileCryptoMetaData) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { - if err := oprot.WriteFieldBegin(ctx, "encryption_algorithm", thrift.STRUCT, 1); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:encryption_algorithm: ", p), err) } - if err := p.EncryptionAlgorithm.Write(ctx, oprot); err != nil { - return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.EncryptionAlgorithm), err) - } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 1:encryption_algorithm: ", p), err) } - return err + if err := oprot.WriteFieldBegin(ctx, "encryption_algorithm", thrift.STRUCT, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:encryption_algorithm: ", p), err) + } + if err := p.EncryptionAlgorithm.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.EncryptionAlgorithm), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:encryption_algorithm: ", p), err) + } + return err } func (p *FileCryptoMetaData) writeField2(ctx context.Context, oprot thrift.TProtocol) (err error) { - if p.IsSetKeyMetadata() { - if err := oprot.WriteFieldBegin(ctx, "key_metadata", thrift.STRING, 2); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:key_metadata: ", p), err) } - if err := oprot.WriteBinary(ctx, p.KeyMetadata); err != nil { - return thrift.PrependError(fmt.Sprintf("%T.key_metadata (2) field write error: ", p), err) } - if err := oprot.WriteFieldEnd(ctx); err != nil { - return thrift.PrependError(fmt.Sprintf("%T write field end error 2:key_metadata: ", p), err) } - } - return err + if p.IsSetKeyMetadata() { + if err := oprot.WriteFieldBegin(ctx, "key_metadata", thrift.STRING, 2); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 2:key_metadata: ", p), err) + } + if err := oprot.WriteBinary(ctx, p.KeyMetadata); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.key_metadata (2) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 2:key_metadata: ", p), err) + } + } + return err } func (p *FileCryptoMetaData) Equals(other *FileCryptoMetaData) bool { - if p == other { - return true - } else if p == nil || other == nil { - return false - } - if !p.EncryptionAlgorithm.Equals(other.EncryptionAlgorithm) { return false } - if bytes.Compare(p.KeyMetadata, other.KeyMetadata) != 0 { return false } - return true + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if !p.EncryptionAlgorithm.Equals(other.EncryptionAlgorithm) { + return false + } + if bytes.Compare(p.KeyMetadata, other.KeyMetadata) != 0 { + return false + } + return true } func (p *FileCryptoMetaData) String() string { - if p == nil { - return "" - } - return fmt.Sprintf("FileCryptoMetaData(%+v)", *p) + if p == nil { + return "" + } + return fmt.Sprintf("FileCryptoMetaData(%+v)", *p) } func (p *FileCryptoMetaData) Validate() error { - return nil + return nil } diff --git a/go/parquet/internal/utils/bit_packing_avx2_amd64.go b/go/parquet/internal/utils/bit_packing_avx2_amd64.go index ee01f002b5ece..0455ccc505bfe 100644 --- a/go/parquet/internal/utils/bit_packing_avx2_amd64.go +++ b/go/parquet/internal/utils/bit_packing_avx2_amd64.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !noasm // +build !noasm package utils diff --git a/go/parquet/internal/utils/bit_packing_neon_arm64.go b/go/parquet/internal/utils/bit_packing_neon_arm64.go index 8d09c891155ef..09154e3e4b7dd 100755 --- a/go/parquet/internal/utils/bit_packing_neon_arm64.go +++ b/go/parquet/internal/utils/bit_packing_neon_arm64.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !noasm // +build !noasm package utils diff --git a/go/parquet/internal/utils/unpack_bool_amd64.go b/go/parquet/internal/utils/unpack_bool_amd64.go index 1e9680db4b21a..2b2054f3b00b8 100644 --- a/go/parquet/internal/utils/unpack_bool_amd64.go +++ b/go/parquet/internal/utils/unpack_bool_amd64.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !noasm // +build !noasm package utils diff --git a/go/parquet/internal/utils/unpack_bool_arm64.go b/go/parquet/internal/utils/unpack_bool_arm64.go index 2c3b19eca458b..879ffd3c9540d 100644 --- a/go/parquet/internal/utils/unpack_bool_arm64.go +++ b/go/parquet/internal/utils/unpack_bool_arm64.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !noasm // +build !noasm package utils @@ -21,13 +22,14 @@ package utils import ( "os" "strings" + + "golang.org/x/sys/cpu" ) -import "golang.org/x/sys/cpu" var byteToBoolFunc func([]byte, []bool) func init() { - // Added ability to enable extension via environment: + // Added ability to enable extension via environment: // ARM_ENABLE_EXT=NEON go test if ext, ok := os.LookupEnv("ARM_ENABLE_EXT"); ok { exts := strings.Split(ext, ",") diff --git a/go/parquet/internal/utils/unpack_bool_avx2_amd64.go b/go/parquet/internal/utils/unpack_bool_avx2_amd64.go index e0065e5aad16d..cec772a2ccf97 100644 --- a/go/parquet/internal/utils/unpack_bool_avx2_amd64.go +++ b/go/parquet/internal/utils/unpack_bool_avx2_amd64.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !noasm // +build !noasm package utils diff --git a/go/parquet/internal/utils/unpack_bool_neon_arm64.go b/go/parquet/internal/utils/unpack_bool_neon_arm64.go index 2e9808abbf157..ed46ce29e0309 100755 --- a/go/parquet/internal/utils/unpack_bool_neon_arm64.go +++ b/go/parquet/internal/utils/unpack_bool_neon_arm64.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !noasm // +build !noasm package utils diff --git a/go/parquet/internal/utils/unpack_bool_noasm.go b/go/parquet/internal/utils/unpack_bool_noasm.go index a715366c6418d..eba20fa9c0f56 100644 --- a/go/parquet/internal/utils/unpack_bool_noasm.go +++ b/go/parquet/internal/utils/unpack_bool_noasm.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build noasm // +build noasm package utils diff --git a/go/parquet/internal/utils/unpack_bool_sse4_amd64.go b/go/parquet/internal/utils/unpack_bool_sse4_amd64.go index 85e4aa77df73b..d00c37474e61c 100644 --- a/go/parquet/internal/utils/unpack_bool_sse4_amd64.go +++ b/go/parquet/internal/utils/unpack_bool_sse4_amd64.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !noasm // +build !noasm package utils diff --git a/go/parquet/metadata/app_version.go b/go/parquet/metadata/app_version.go index f61c4c9703f78..fa54aec347575 100644 --- a/go/parquet/metadata/app_version.go +++ b/go/parquet/metadata/app_version.go @@ -74,7 +74,8 @@ func NewAppVersionExplicit(app string, major, minor, patch int) *AppVersion { // NewAppVersion parses a "created by" string such as "parquet-go 1.0.0". // // It also supports handling pre-releases and build info such as -// parquet-cpp version 1.5.0ab-xyz5.5.0+cd (build abcd) +// +// parquet-cpp version 1.5.0ab-xyz5.5.0+cd (build abcd) func NewAppVersion(createdby string) *AppVersion { v := &AppVersion{} diff --git a/go/parquet/schema/reflection.go b/go/parquet/schema/reflection.go index 1ec9c72f3dfc1..f961c6ef26d08 100644 --- a/go/parquet/schema/reflection.go +++ b/go/parquet/schema/reflection.go @@ -551,7 +551,7 @@ func typeToNode(name string, typ reflect.Type, repType parquet.Repetition, info // NewSchemaFromStruct generates a schema from an object type via reflection of // the type and reading struct tags for "parquet". // -// Rules +// # Rules // // Everything defaults to Required repetition, unless otherwise specified. // Pointer types become Optional repetition. @@ -571,7 +571,7 @@ func typeToNode(name string, typ reflect.Type, repType parquet.Repetition, info // // maps will become appropriate Map structures in the schema of the defined key and values. // -// Available Tags +// # Available Tags // // name: by default the node will have the same name as the field, this tag let's you specify a name // diff --git a/go/parquet/tools.go b/go/parquet/tools.go index b9ce84def5ae0..64e9419e4f711 100644 --- a/go/parquet/tools.go +++ b/go/parquet/tools.go @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build tools // +build tools package tools diff --git a/swift/data-generator/swift-datagen/main.go b/swift/data-generator/swift-datagen/main.go index a60fb562932fe..2f2e244ab5891 100644 --- a/swift/data-generator/swift-datagen/main.go +++ b/swift/data-generator/swift-datagen/main.go @@ -22,8 +22,8 @@ import ( "github.com/apache/arrow/go/v12/arrow" "github.com/apache/arrow/go/v12/arrow/array" - "github.com/apache/arrow/go/v12/arrow/memory" "github.com/apache/arrow/go/v12/arrow/ipc" + "github.com/apache/arrow/go/v12/arrow/memory" ) func writeBytes(rec arrow.Record, file_name string) { @@ -42,7 +42,6 @@ func writeBytes(rec arrow.Record, file_name string) { rr.Close() } - func writeBoolData() { alloc := memory.NewGoAllocator() schema := arrow.NewSchema([]arrow.Field{ @@ -53,14 +52,13 @@ func writeBoolData() { b := array.NewRecordBuilder(alloc, schema) defer b.Release() - b.Field(0).(*array.BooleanBuilder).AppendValues([]bool{true, false,}, nil) + b.Field(0).(*array.BooleanBuilder).AppendValues([]bool{true, false}, nil) b.Field(0).(*array.BooleanBuilder).AppendNull() - b.Field(0).(*array.BooleanBuilder).AppendValues([]bool{false, true,}, nil) + b.Field(0).(*array.BooleanBuilder).AppendValues([]bool{false, true}, nil) b.Field(1).(*array.StringBuilder).AppendValues([]string{"zero", "one", "two", "three", "four"}, nil) rec := b.NewRecord() defer rec.Release() - writeBytes(rec, "testdata_bool.arrow") } @@ -81,11 +79,10 @@ func writeDoubleData() { rec := b.NewRecord() defer rec.Release() - writeBytes(rec, "testdata_double.arrow") } func main() { - writeBoolData(); - writeDoubleData(); + writeBoolData() + writeDoubleData() } From 813fe2596751fe9577dbe9beca2c50a351c4c2dd Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Mon, 3 Jun 2024 18:18:38 +0100 Subject: [PATCH 41/93] GH-41829: [R] Update relative URLs in README to absolute paths to prevent CRAN check failures (#41830) ### Rationale for this change Relative URLs in README mean we fail CRAN checks ### What changes are included in this PR? Update relative URLs to absolute URLs ### Are these changes tested? Nope ### Are there any user-facing changes? Nope * GitHub Issue: #41829 Authored-by: Nic Crane Signed-off-by: Nic Crane --- r/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/r/README.md b/r/README.md index 710fa8e8d7cb5..c3cd5a32eaf69 100644 --- a/r/README.md +++ b/r/README.md @@ -12,7 +12,7 @@ The R `{arrow}` package provides access to many of the features of the [Apache Arrow C++ library](https://arrow.apache.org/docs/cpp/index.html) for R users. The goal of arrow is to provide an Arrow C++ backend to `{dplyr}`, and access to the Arrow C++ library through familiar base R and tidyverse functions, or `{R6}` classes. -To learn more about the Apache Arrow project, see the parent documentation of the [Arrow Project](https://arrow.apache.org/). The Arrow project provides functionality for a wide range of data analysis tasks to store, process and move data fast. See the [read/write article](articles/read_write.html) to learn about reading and writing data files, [data wrangling](articles/data_wrangling.html) to learn how to use dplyr syntax with arrow objects, and the [function documentation](reference/acero.html) for a full list of supported functions within dplyr queries. +To learn more about the Apache Arrow project, see the parent documentation of the [Arrow Project](https://arrow.apache.org/). The Arrow project provides functionality for a wide range of data analysis tasks to store, process and move data fast. See the [read/write article](https://arrow.apache.org/docs/r/articles/read_write.html) to learn about reading and writing data files, [data wrangling](https://arrow.apache.org/docs/r/articles/data_wrangling.html) to learn how to use dplyr syntax with arrow objects, and the [function documentation](https://arrow.apache.org/docs/r/reference/acero.html) for a full list of supported functions within dplyr queries. ## Installation @@ -33,11 +33,11 @@ There are some special cases to note: - On macOS, the R you use with Arrow should match the architecture of the machine you are using. If you're using an ARM (aka M1, M2, etc.) processor use R compiled for arm64. If you're using an Intel based mac, use R compiled for x86. Using R and Arrow compiled for Intel based macs on an ARM based mac will result in segfaults and crashes. -- On Linux the installation process can sometimes be more involved because CRAN does not host binaries for Linux. For more information please see the [installation guide](articles/install.html). +- On Linux the installation process can sometimes be more involved because CRAN does not host binaries for Linux. For more information please see the [installation guide](https://arrow.apache.org/docs/r/articles/install.html). - If you are compiling arrow from source, please note that as of version 10.0.0, arrow requires C++17 to build. This has implications on Windows and CentOS 7. For Windows users it means you need to be running an R version of 4.0 or later. On CentOS 7, it means you need to install a newer compiler than the default system compiler gcc. See the [installation details article](https://arrow.apache.org/docs/r/articles/developers/install_details.html) for guidance. -- Development versions of arrow are released nightly. For information on how to installl nighhtly builds please see the [installing nightly builds](articles/install_nightly.html) article. +- Development versions of arrow are released nightly. For information on how to installl nightly builds please see the [installing nightly builds](https://arrow.apache.org/docs/r/articles/install_nightly.html) article. ## What can the arrow package do? From 1598782d4ffd3f7a961f379148f17f34e16caf2b Mon Sep 17 00:00:00 2001 From: Steve Lord <72518652+stevelorddremio@users.noreply.github.com> Date: Mon, 3 Jun 2024 12:57:42 -0700 Subject: [PATCH 42/93] GH-41262: [Java][FlightSQL] Implement stateless prepared statements (#41237) ### Rationale for this change Expand the number of implemented languages for stateless prepared statements to include Java. ### What changes are included in this PR? Update FlightSqlClient and include a stateless server implementation example with tests. ### Are these changes tested? Yes, tests are added to cover a stateless server implementation. ### Are there any user-facing changes? There is a modified FlightSqlClient that is required to enable use of stateless prepared statements. * GitHub Issue: #41262 Lead-authored-by: Steve Lord Co-authored-by: Mateusz Rzeszutek Signed-off-by: David Li --- .../arrow/flight/sql/FlightSqlClient.java | 27 +- .../DoPutPreparedStatementResultPOJO.java | 38 +++ .../flight/sql/example/FlightSqlExample.java | 60 +++-- .../example/FlightSqlStatelessExample.java | 238 ++++++++++++++++++ .../arrow/flight/sql/test/TestFlightSql.java | 63 +++-- .../sql/test/TestFlightSqlStateless.java | 99 ++++++++ 6 files changed, 474 insertions(+), 51 deletions(-) create mode 100644 java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/example/DoPutPreparedStatementResultPOJO.java create mode 100644 java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/example/FlightSqlStatelessExample.java create mode 100644 java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStateless.java diff --git a/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlClient.java b/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlClient.java index 6fe31fae9216b..a94dc563cfbcc 100644 --- a/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlClient.java +++ b/java/flight/flight-sql/src/main/java/org/apache/arrow/flight/sql/FlightSqlClient.java @@ -78,6 +78,7 @@ import org.apache.arrow.flight.SetSessionOptionsResult; import org.apache.arrow.flight.SyncPutListener; import org.apache.arrow.flight.Ticket; +import org.apache.arrow.flight.sql.impl.FlightSql; import org.apache.arrow.flight.sql.impl.FlightSql.ActionCreatePreparedStatementResult; import org.apache.arrow.flight.sql.impl.FlightSql.CommandPreparedStatementQuery; import org.apache.arrow.flight.sql.util.TableRef; @@ -1048,15 +1049,35 @@ private Schema deserializeSchema(final ByteString bytes) { public FlightInfo execute(final CallOption... options) { checkOpen(); - final FlightDescriptor descriptor = FlightDescriptor + FlightDescriptor descriptor = FlightDescriptor .command(Any.pack(CommandPreparedStatementQuery.newBuilder() .setPreparedStatementHandle(preparedStatementResult.getPreparedStatementHandle()) .build()) .toByteArray()); if (parameterBindingRoot != null && parameterBindingRoot.getRowCount() > 0) { - try (final SyncPutListener listener = putParameters(descriptor, options)) { - listener.getResult(); + try (final SyncPutListener putListener = putParameters(descriptor, options)) { + if (getParameterSchema().getFields().size() > 0 && + parameterBindingRoot != null && + parameterBindingRoot.getRowCount() > 0) { + final PutResult read = putListener.read(); + if (read != null) { + try (final ArrowBuf metadata = read.getApplicationMetadata()) { + final FlightSql.DoPutPreparedStatementResult doPutPreparedStatementResult = + FlightSql.DoPutPreparedStatementResult.parseFrom(metadata.nioBuffer()); + descriptor = FlightDescriptor + .command(Any.pack(CommandPreparedStatementQuery.newBuilder() + .setPreparedStatementHandle( + doPutPreparedStatementResult.getPreparedStatementHandle()) + .build()) + .toByteArray()); + } + } + } + } catch (final InterruptedException | ExecutionException e) { + throw CallStatus.CANCELLED.withCause(e).toRuntimeException(); + } catch (final InvalidProtocolBufferException e) { + throw CallStatus.INVALID_ARGUMENT.withCause(e).toRuntimeException(); } } diff --git a/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/example/DoPutPreparedStatementResultPOJO.java b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/example/DoPutPreparedStatementResultPOJO.java new file mode 100644 index 0000000000000..ace78862b014d --- /dev/null +++ b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/example/DoPutPreparedStatementResultPOJO.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.flight.sql.example; + +import java.io.Serializable; + +public class DoPutPreparedStatementResultPOJO implements Serializable { + private String query; + private byte[] parameters; + + public DoPutPreparedStatementResultPOJO(String query, byte[] parameters) { + this.query = query; + this.parameters = parameters.clone(); + } + + public String getQuery() { + return query; + } + + public byte[] getParameters() { + return parameters; + } +} diff --git a/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/example/FlightSqlExample.java b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/example/FlightSqlExample.java index 52c402efd6f0b..36362fd8681d3 100644 --- a/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/example/FlightSqlExample.java +++ b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/example/FlightSqlExample.java @@ -156,21 +156,22 @@ * supports all current features of Flight SQL. */ public class FlightSqlExample implements FlightSqlProducer, AutoCloseable { - private static final String DATABASE_URI = "jdbc:derby:target/derbyDB"; private static final Logger LOGGER = getLogger(FlightSqlExample.class); - private static final Calendar DEFAULT_CALENDAR = JdbcToArrowUtils.getUtcCalendar(); + protected static final Calendar DEFAULT_CALENDAR = JdbcToArrowUtils.getUtcCalendar(); + public static final String DB_NAME = "derbyDB"; + private final String databaseUri; // ARROW-15315: Use ExecutorService to simulate an async scenario private final ExecutorService executorService = Executors.newFixedThreadPool(10); private final Location location; - private final PoolingDataSource dataSource; - private final BufferAllocator rootAllocator = new RootAllocator(); + protected final PoolingDataSource dataSource; + protected final BufferAllocator rootAllocator = new RootAllocator(); private final Cache> preparedStatementLoadingCache; private final Cache> statementLoadingCache; private final SqlInfoBuilder sqlInfoBuilder; public static void main(String[] args) throws Exception { Location location = Location.forGrpcInsecure("localhost", 55555); - final FlightSqlExample example = new FlightSqlExample(location); + final FlightSqlExample example = new FlightSqlExample(location, DB_NAME); Location listenLocation = Location.forGrpcInsecure("0.0.0.0", 55555); try (final BufferAllocator allocator = new RootAllocator(); final FlightServer server = FlightServer.builder(allocator, listenLocation, example).build()) { @@ -179,13 +180,14 @@ public static void main(String[] args) throws Exception { } } - public FlightSqlExample(final Location location) { + public FlightSqlExample(final Location location, final String dbName) { // TODO Constructor should not be doing work. checkState( - removeDerbyDatabaseIfExists() && populateDerbyDatabase(), + removeDerbyDatabaseIfExists(dbName) && populateDerbyDatabase(dbName), "Failed to reset Derby database!"); + databaseUri = "jdbc:derby:target/" + dbName; final ConnectionFactory connectionFactory = - new DriverManagerConnectionFactory(DATABASE_URI, new Properties()); + new DriverManagerConnectionFactory(databaseUri, new Properties()); final PoolableConnectionFactory poolableConnectionFactory = new PoolableConnectionFactory(connectionFactory, null); final ObjectPool connectionPool = new GenericObjectPool<>(poolableConnectionFactory); @@ -248,9 +250,9 @@ public FlightSqlExample(final Location location) { } - private static boolean removeDerbyDatabaseIfExists() { + public static boolean removeDerbyDatabaseIfExists(final String dbName) { boolean wasSuccess; - final Path path = Paths.get("target" + File.separator + "derbyDB"); + final Path path = Paths.get("target" + File.separator + dbName); try (final Stream walk = Files.walk(path)) { /* @@ -262,7 +264,7 @@ private static boolean removeDerbyDatabaseIfExists() { * this not expected. */ wasSuccess = walk.sorted(Comparator.reverseOrder()).map(Path::toFile).map(File::delete) - .reduce(Boolean::logicalAnd).orElseThrow(IOException::new); + .reduce(Boolean::logicalAnd).orElseThrow(IOException::new); } catch (IOException e) { /* * The only acceptable scenario for an `IOException` to be thrown here is if @@ -277,9 +279,12 @@ private static boolean removeDerbyDatabaseIfExists() { return wasSuccess; } - private static boolean populateDerbyDatabase() { - try (final Connection connection = DriverManager.getConnection("jdbc:derby:target/derbyDB;create=true"); + private static boolean populateDerbyDatabase(final String dbName) { + try (final Connection connection = DriverManager.getConnection("jdbc:derby:target/" + dbName + ";create=true"); Statement statement = connection.createStatement()) { + + dropTable(statement, "intTable"); + dropTable(statement, "foreignTable"); statement.execute("CREATE TABLE foreignTable (" + "id INT not null primary key GENERATED ALWAYS AS IDENTITY (START WITH 1, INCREMENT BY 1), " + "foreignName varchar(100), " + @@ -302,6 +307,18 @@ private static boolean populateDerbyDatabase() { return true; } + private static void dropTable(final Statement statement, final String tableName) throws SQLException { + try { + statement.execute("DROP TABLE " + tableName); + } catch (SQLException e) { + // sql error code for "object does not exist"; which is fine, we're trying to delete the table + // see https://db.apache.org/derby/docs/10.17/ref/rrefexcept71493.html + if (!"42Y55".equals(e.getSQLState())) { + throw e; + } + } + } + private static ArrowType getArrowTypeFromJdbcType(final int jdbcDataType, final int precision, final int scale) { try { return JdbcToArrowUtils.getArrowTypeFromJdbcType(new JdbcFieldInfo(jdbcDataType, precision, scale), @@ -778,7 +795,7 @@ public void createPreparedStatement(final ActionCreatePreparedStatementRequest r // Running on another thread Future unused = executorService.submit(() -> { try { - final ByteString preparedStatementHandle = copyFrom(randomUUID().toString().getBytes(StandardCharsets.UTF_8)); + final ByteString preparedStatementHandle = copyFrom(request.getQuery().getBytes(StandardCharsets.UTF_8)); // Ownership of the connection will be passed to the context. Do NOT close! final Connection connection = dataSource.getConnection(); final PreparedStatement preparedStatement = connection.prepareStatement(request.getQuery(), @@ -882,7 +899,7 @@ public Runnable acceptPutPreparedStatementUpdate(CommandPreparedStatementUpdate while (binder.next()) { preparedStatement.addBatch(); } - int[] recordCounts = preparedStatement.executeBatch(); + final int[] recordCounts = preparedStatement.executeBatch(); recordCount = Arrays.stream(recordCounts).sum(); } @@ -928,6 +945,7 @@ public Runnable acceptPutPreparedStatementQuery(CommandPreparedStatementQuery co .toRuntimeException()); return; } + ackStream.onCompleted(); }; } @@ -1035,7 +1053,7 @@ public void getStreamTables(final CommandGetTables command, final CallContext co final String[] tableTypes = protocolSize == 0 ? null : protocolStringList.toArray(new String[protocolSize]); - try (final Connection connection = DriverManager.getConnection(DATABASE_URI); + try (final Connection connection = DriverManager.getConnection(databaseUri); final VectorSchemaRoot vectorSchemaRoot = getTablesRoot( connection.getMetaData(), rootAllocator, @@ -1086,7 +1104,7 @@ public void getStreamPrimaryKeys(final CommandGetPrimaryKeys command, final Call final String schema = command.hasDbSchema() ? command.getDbSchema() : null; final String table = command.getTable(); - try (Connection connection = DriverManager.getConnection(DATABASE_URI)) { + try (Connection connection = DriverManager.getConnection(databaseUri)) { final ResultSet primaryKeys = connection.getMetaData().getPrimaryKeys(catalog, schema, table); final VarCharVector catalogNameVector = new VarCharVector("catalog_name", rootAllocator); @@ -1140,7 +1158,7 @@ public void getStreamExportedKeys(final CommandGetExportedKeys command, final Ca String schema = command.hasDbSchema() ? command.getDbSchema() : null; String table = command.getTable(); - try (Connection connection = DriverManager.getConnection(DATABASE_URI); + try (Connection connection = DriverManager.getConnection(databaseUri); ResultSet keys = connection.getMetaData().getExportedKeys(catalog, schema, table); VectorSchemaRoot vectorSchemaRoot = createVectors(keys)) { listener.start(vectorSchemaRoot); @@ -1165,7 +1183,7 @@ public void getStreamImportedKeys(final CommandGetImportedKeys command, final Ca String schema = command.hasDbSchema() ? command.getDbSchema() : null; String table = command.getTable(); - try (Connection connection = DriverManager.getConnection(DATABASE_URI); + try (Connection connection = DriverManager.getConnection(databaseUri); ResultSet keys = connection.getMetaData().getImportedKeys(catalog, schema, table); VectorSchemaRoot vectorSchemaRoot = createVectors(keys)) { listener.start(vectorSchemaRoot); @@ -1193,7 +1211,7 @@ public void getStreamCrossReference(CommandGetCrossReference command, CallContex final String pkTable = command.getPkTable(); final String fkTable = command.getFkTable(); - try (Connection connection = DriverManager.getConnection(DATABASE_URI); + try (Connection connection = DriverManager.getConnection(databaseUri); ResultSet keys = connection.getMetaData() .getCrossReference(pkCatalog, pkSchema, pkTable, fkCatalog, fkSchema, fkTable); VectorSchemaRoot vectorSchemaRoot = createVectors(keys)) { @@ -1280,7 +1298,7 @@ public void getStreamStatement(final TicketStatementQuery ticketStatementQuery, } } - private FlightInfo getFlightInfoForSchema(final T request, final FlightDescriptor descriptor, + protected FlightInfo getFlightInfoForSchema(final T request, final FlightDescriptor descriptor, final Schema schema) { final Ticket ticket = new Ticket(pack(request).toByteArray()); // TODO Support multiple endpoints. diff --git a/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/example/FlightSqlStatelessExample.java b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/example/FlightSqlStatelessExample.java new file mode 100644 index 0000000000000..c79c09c0967dc --- /dev/null +++ b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/example/FlightSqlStatelessExample.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.flight.sql.example; + +import static java.lang.String.format; +import static org.apache.arrow.adapter.jdbc.JdbcToArrow.sqlToArrowVectorIterator; +import static org.apache.arrow.adapter.jdbc.JdbcToArrowUtils.jdbcToArrowSchema; +import static org.apache.arrow.flight.sql.impl.FlightSql.*; +import static org.slf4j.LoggerFactory.getLogger; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.StreamCorruptedException; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; + +import org.apache.arrow.adapter.jdbc.ArrowVectorIterator; +import org.apache.arrow.adapter.jdbc.JdbcParameterBinder; +import org.apache.arrow.flight.CallStatus; +import org.apache.arrow.flight.FlightDescriptor; +import org.apache.arrow.flight.FlightInfo; +import org.apache.arrow.flight.FlightStream; +import org.apache.arrow.flight.Location; +import org.apache.arrow.flight.PutResult; +import org.apache.arrow.flight.sql.FlightSqlProducer; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.vector.VectorLoader; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.VectorUnloader; +import org.apache.arrow.vector.ipc.ArrowFileReader; +import org.apache.arrow.vector.ipc.ArrowFileWriter; +import org.apache.arrow.vector.ipc.SeekableReadChannel; +import org.apache.arrow.vector.ipc.message.ArrowBlock; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel; +import org.slf4j.Logger; + +import com.google.protobuf.ByteString; + +/** + * Example {@link FlightSqlProducer} implementation showing an Apache Derby backed Flight SQL server that generally + * supports all current features of Flight SQL. + */ +public class FlightSqlStatelessExample extends FlightSqlExample { + private static final Logger LOGGER = getLogger(FlightSqlStatelessExample.class); + public static final String DB_NAME = "derbyStatelessDB"; + + + public FlightSqlStatelessExample(final Location location, final String dbName) { + super(location, dbName); + } + + @Override + public Runnable acceptPutPreparedStatementQuery(CommandPreparedStatementQuery command, CallContext context, + FlightStream flightStream, StreamListener ackStream) { + + return () -> { + final String query = new String(command.getPreparedStatementHandle().toStringUtf8()); + try (Connection connection = dataSource.getConnection(); + PreparedStatement preparedStatement = createPreparedStatement(connection, query)) { + while (flightStream.next()) { + final VectorSchemaRoot root = flightStream.getRoot(); + final JdbcParameterBinder binder = JdbcParameterBinder.builder(preparedStatement, root).bindAll().build(); + while (binder.next()) { + // Do not execute() - will be done in a getStream call + } + + final ByteArrayOutputStream parametersStream = new ByteArrayOutputStream(); + try (ArrowFileWriter writer = new ArrowFileWriter(root, null, Channels.newChannel(parametersStream)) + ) { + writer.start(); + writer.writeBatch(); + } + + if (parametersStream.size() > 0) { + final DoPutPreparedStatementResultPOJO doPutPreparedStatementResultPOJO = + new DoPutPreparedStatementResultPOJO(query, parametersStream.toByteArray()); + + final byte[] doPutPreparedStatementResultPOJOArr = serializePOJO(doPutPreparedStatementResultPOJO); + final DoPutPreparedStatementResult doPutPreparedStatementResult = + DoPutPreparedStatementResult.newBuilder() + .setPreparedStatementHandle( + ByteString.copyFrom(ByteBuffer.wrap(doPutPreparedStatementResultPOJOArr))) + .build(); + + try (final ArrowBuf buffer = rootAllocator.buffer(doPutPreparedStatementResult.getSerializedSize())) { + buffer.writeBytes(doPutPreparedStatementResult.toByteArray()); + ackStream.onNext(PutResult.metadata(buffer)); + } + } + } + + } catch (SQLException | IOException e) { + ackStream.onError(CallStatus.INTERNAL + .withDescription("Failed to bind parameters: " + e.getMessage()) + .withCause(e) + .toRuntimeException()); + return; + } + + ackStream.onCompleted(); + }; + } + + @Override + public void getStreamPreparedStatement(final CommandPreparedStatementQuery command, final CallContext context, + final ServerStreamListener listener) { + final byte[] handle = command.getPreparedStatementHandle().toByteArray(); + try { + // Case where there are parameters + try { + final DoPutPreparedStatementResultPOJO doPutPreparedStatementResultPOJO = + deserializePOJO(handle); + final String query = doPutPreparedStatementResultPOJO.getQuery(); + + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = createPreparedStatement(connection, query); + ArrowFileReader reader = new ArrowFileReader(new SeekableReadChannel( + new ByteArrayReadableSeekableByteChannel( + doPutPreparedStatementResultPOJO.getParameters())), rootAllocator)) { + + for (ArrowBlock arrowBlock : reader.getRecordBlocks()) { + reader.loadRecordBatch(arrowBlock); + VectorSchemaRoot vectorSchemaRootRecover = reader.getVectorSchemaRoot(); + JdbcParameterBinder binder = JdbcParameterBinder.builder(statement, vectorSchemaRootRecover) + .bindAll().build(); + + while (binder.next()) { + executeQuery(statement, listener); + } + } + } + } catch (StreamCorruptedException e) { + // Case where there are no parameters + final String query = new String(command.getPreparedStatementHandle().toStringUtf8()); + try (Connection connection = dataSource.getConnection(); + PreparedStatement preparedStatement = createPreparedStatement(connection, query)) { + executeQuery(preparedStatement, listener); + } + } + } catch (final SQLException | IOException | ClassNotFoundException e) { + LOGGER.error(format("Failed to getStreamPreparedStatement: <%s>.", e.getMessage()), e); + listener.error(CallStatus.INTERNAL.withDescription("Failed to prepare statement: " + e).toRuntimeException()); + } finally { + listener.completed(); + } + } + + private void executeQuery(PreparedStatement statement, + final ServerStreamListener listener) throws IOException, SQLException { + try (final ResultSet resultSet = statement.executeQuery()) { + final Schema schema = jdbcToArrowSchema(resultSet.getMetaData(), DEFAULT_CALENDAR); + try (final VectorSchemaRoot vectorSchemaRoot = VectorSchemaRoot.create(schema, rootAllocator)) { + final VectorLoader loader = new VectorLoader(vectorSchemaRoot); + listener.start(vectorSchemaRoot); + + final ArrowVectorIterator iterator = sqlToArrowVectorIterator(resultSet, rootAllocator); + while (iterator.hasNext()) { + final VectorSchemaRoot batch = iterator.next(); + if (batch.getRowCount() == 0) { + break; + } + final VectorUnloader unloader = new VectorUnloader(batch); + loader.load(unloader.getRecordBatch()); + listener.putNext(); + vectorSchemaRoot.clear(); + } + listener.putNext(); + } + } + } + + @Override + public FlightInfo getFlightInfoPreparedStatement(final CommandPreparedStatementQuery command, + final CallContext context, + final FlightDescriptor descriptor) { + final byte[] handle = command.getPreparedStatementHandle().toByteArray(); + try { + String query; + try { + query = deserializePOJO(handle).getQuery(); + } catch (StreamCorruptedException e) { + query = new String(command.getPreparedStatementHandle().toStringUtf8()); + } + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = createPreparedStatement(connection, query)) { + ResultSetMetaData metaData = statement.getMetaData(); + return getFlightInfoForSchema(command, descriptor, + jdbcToArrowSchema(metaData, DEFAULT_CALENDAR)); + } + } catch (final SQLException | IOException | ClassNotFoundException e) { + LOGGER.error(format("There was a problem executing the prepared statement: <%s>.", e.getMessage()), e); + throw CallStatus.INTERNAL.withCause(e).toRuntimeException(); + } + } + + private DoPutPreparedStatementResultPOJO deserializePOJO(byte[] handle) throws IOException, ClassNotFoundException { + try (ByteArrayInputStream bis = new ByteArrayInputStream(handle); + ObjectInputStream ois = new ObjectInputStream(bis)) { + return (DoPutPreparedStatementResultPOJO) ois.readObject(); + } + } + + private byte[] serializePOJO(DoPutPreparedStatementResultPOJO doPutPreparedStatementResultPOJO) throws IOException { + try (ByteArrayOutputStream bos = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream(bos)) { + oos.writeObject(doPutPreparedStatementResultPOJO); + return bos.toByteArray(); + } + } + + private PreparedStatement createPreparedStatement(Connection connection, String query) throws SQLException { + return connection.prepareStatement(query, ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY); + } +} diff --git a/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSql.java b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSql.java index a39736e939f0b..ffffdd62ac950 100644 --- a/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSql.java +++ b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSql.java @@ -87,63 +87,72 @@ public class TestFlightSql { Field.nullable("FOREIGNID", MinorType.INT.getType()))); private static final List> EXPECTED_RESULTS_FOR_STAR_SELECT_QUERY = ImmutableList.of( asList("1", "one", "1", "1"), asList("2", "zero", "0", "1"), asList("3", "negative one", "-1", "1")); - private static final List> EXPECTED_RESULTS_FOR_PARAMETER_BINDING = ImmutableList.of( + protected static final List> EXPECTED_RESULTS_FOR_PARAMETER_BINDING = ImmutableList.of( asList("1", "one", "1", "1")); private static final Map GET_SQL_INFO_EXPECTED_RESULTS_MAP = new LinkedHashMap<>(); - private static final String LOCALHOST = "localhost"; - private static BufferAllocator allocator; - private static FlightServer server; - private static FlightSqlClient sqlClient; + protected static final String LOCALHOST = "localhost"; + protected static BufferAllocator allocator; + protected static FlightServer server; + protected static FlightSqlClient sqlClient; @BeforeAll public static void setUp() throws Exception { + setUpClientServer(); + setUpExpectedResultsMap(); + } + + private static void setUpClientServer() throws Exception { allocator = new RootAllocator(Integer.MAX_VALUE); final Location serverLocation = Location.forGrpcInsecure(LOCALHOST, 0); - server = FlightServer.builder(allocator, serverLocation, new FlightSqlExample(serverLocation)) - .build() - .start(); + server = FlightServer.builder(allocator, serverLocation, + new FlightSqlExample(serverLocation, FlightSqlExample.DB_NAME)) + .build() + .start(); final Location clientLocation = Location.forGrpcInsecure(LOCALHOST, server.getPort()); sqlClient = new FlightSqlClient(FlightClient.builder(allocator, clientLocation).build()); + } + protected static void setUpExpectedResultsMap() { GET_SQL_INFO_EXPECTED_RESULTS_MAP - .put(Integer.toString(FlightSql.SqlInfo.FLIGHT_SQL_SERVER_NAME_VALUE), "Apache Derby"); + .put(Integer.toString(FlightSql.SqlInfo.FLIGHT_SQL_SERVER_NAME_VALUE), "Apache Derby"); GET_SQL_INFO_EXPECTED_RESULTS_MAP - .put(Integer.toString(FlightSql.SqlInfo.FLIGHT_SQL_SERVER_VERSION_VALUE), "10.14.2.0 - (1828579)"); + .put(Integer.toString(FlightSql.SqlInfo.FLIGHT_SQL_SERVER_VERSION_VALUE), "10.14.2.0 - (1828579)"); GET_SQL_INFO_EXPECTED_RESULTS_MAP - .put(Integer.toString(FlightSql.SqlInfo.FLIGHT_SQL_SERVER_ARROW_VERSION_VALUE), "10.14.2.0 - (1828579)"); + .put(Integer.toString(FlightSql.SqlInfo.FLIGHT_SQL_SERVER_ARROW_VERSION_VALUE), "10.14.2.0 - (1828579)"); GET_SQL_INFO_EXPECTED_RESULTS_MAP - .put(Integer.toString(FlightSql.SqlInfo.FLIGHT_SQL_SERVER_READ_ONLY_VALUE), "false"); + .put(Integer.toString(FlightSql.SqlInfo.FLIGHT_SQL_SERVER_READ_ONLY_VALUE), "false"); GET_SQL_INFO_EXPECTED_RESULTS_MAP - .put(Integer.toString(FlightSql.SqlInfo.SQL_ALL_TABLES_ARE_SELECTABLE_VALUE), "true"); + .put(Integer.toString(FlightSql.SqlInfo.SQL_ALL_TABLES_ARE_SELECTABLE_VALUE), "true"); GET_SQL_INFO_EXPECTED_RESULTS_MAP - .put( - Integer.toString(FlightSql.SqlInfo.SQL_NULL_ORDERING_VALUE), - Integer.toString(FlightSql.SqlNullOrdering.SQL_NULLS_SORTED_AT_END_VALUE)); + .put( + Integer.toString(FlightSql.SqlInfo.SQL_NULL_ORDERING_VALUE), + Integer.toString(FlightSql.SqlNullOrdering.SQL_NULLS_SORTED_AT_END_VALUE)); GET_SQL_INFO_EXPECTED_RESULTS_MAP - .put(Integer.toString(FlightSql.SqlInfo.SQL_DDL_CATALOG_VALUE), "false"); + .put(Integer.toString(FlightSql.SqlInfo.SQL_DDL_CATALOG_VALUE), "false"); GET_SQL_INFO_EXPECTED_RESULTS_MAP - .put(Integer.toString(FlightSql.SqlInfo.SQL_DDL_SCHEMA_VALUE), "true"); + .put(Integer.toString(FlightSql.SqlInfo.SQL_DDL_SCHEMA_VALUE), "true"); GET_SQL_INFO_EXPECTED_RESULTS_MAP - .put(Integer.toString(FlightSql.SqlInfo.SQL_DDL_TABLE_VALUE), "true"); + .put(Integer.toString(FlightSql.SqlInfo.SQL_DDL_TABLE_VALUE), "true"); GET_SQL_INFO_EXPECTED_RESULTS_MAP - .put( - Integer.toString(FlightSql.SqlInfo.SQL_IDENTIFIER_CASE_VALUE), - Integer.toString(SqlSupportedCaseSensitivity.SQL_CASE_SENSITIVITY_UPPERCASE_VALUE)); + .put( + Integer.toString(FlightSql.SqlInfo.SQL_IDENTIFIER_CASE_VALUE), + Integer.toString(SqlSupportedCaseSensitivity.SQL_CASE_SENSITIVITY_UPPERCASE_VALUE)); GET_SQL_INFO_EXPECTED_RESULTS_MAP - .put(Integer.toString(FlightSql.SqlInfo.SQL_IDENTIFIER_QUOTE_CHAR_VALUE), "\""); + .put(Integer.toString(FlightSql.SqlInfo.SQL_IDENTIFIER_QUOTE_CHAR_VALUE), "\""); GET_SQL_INFO_EXPECTED_RESULTS_MAP - .put( - Integer.toString(FlightSql.SqlInfo.SQL_QUOTED_IDENTIFIER_CASE_VALUE), - Integer.toString(SqlSupportedCaseSensitivity.SQL_CASE_SENSITIVITY_CASE_INSENSITIVE_VALUE)); + .put( + Integer.toString(FlightSql.SqlInfo.SQL_QUOTED_IDENTIFIER_CASE_VALUE), + Integer.toString(SqlSupportedCaseSensitivity.SQL_CASE_SENSITIVITY_CASE_INSENSITIVE_VALUE)); GET_SQL_INFO_EXPECTED_RESULTS_MAP - .put(Integer.toString(FlightSql.SqlInfo.SQL_MAX_COLUMNS_IN_TABLE_VALUE), "42"); + .put(Integer.toString(FlightSql.SqlInfo.SQL_MAX_COLUMNS_IN_TABLE_VALUE), "42"); } @AfterAll public static void tearDown() throws Exception { close(sqlClient, server, allocator); + FlightSqlExample.removeDerbyDatabaseIfExists(FlightSqlExample.DB_NAME); } private static List> getNonConformingResultsForGetSqlInfo(final List> results) { diff --git a/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStateless.java b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStateless.java new file mode 100644 index 0000000000000..09c7b2ef87f45 --- /dev/null +++ b/java/flight/flight-sql/src/test/java/org/apache/arrow/flight/sql/test/TestFlightSqlStateless.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.flight.sql.test; + +import static org.apache.arrow.flight.sql.util.FlightStreamUtils.getResults; +import static org.apache.arrow.util.AutoCloseables.close; +import static org.hamcrest.CoreMatchers.*; + +import org.apache.arrow.flight.FlightClient; +import org.apache.arrow.flight.FlightEndpoint; +import org.apache.arrow.flight.FlightInfo; +import org.apache.arrow.flight.FlightServer; +import org.apache.arrow.flight.FlightStream; +import org.apache.arrow.flight.Location; +import org.apache.arrow.flight.sql.FlightSqlClient; +import org.apache.arrow.flight.sql.FlightSqlClient.PreparedStatement; +import org.apache.arrow.flight.sql.example.FlightSqlStatelessExample; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.Schema; +import org.hamcrest.MatcherAssert; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * Test direct usage of Flight SQL workflows. + */ +public class TestFlightSqlStateless extends TestFlightSql { + + @BeforeAll + public static void setUp() throws Exception { + setUpClientServer(); + setUpExpectedResultsMap(); + } + + @AfterAll + public static void tearDown() throws Exception { + close(sqlClient, server, allocator); + FlightSqlStatelessExample.removeDerbyDatabaseIfExists(FlightSqlStatelessExample.DB_NAME); + } + + private static void setUpClientServer() throws Exception { + allocator = new RootAllocator(Integer.MAX_VALUE); + + final Location serverLocation = Location.forGrpcInsecure(LOCALHOST, 0); + server = FlightServer.builder(allocator, serverLocation, + new FlightSqlStatelessExample(serverLocation, FlightSqlStatelessExample.DB_NAME)) + .build() + .start(); + + final Location clientLocation = Location.forGrpcInsecure(LOCALHOST, server.getPort()); + sqlClient = new FlightSqlClient(FlightClient.builder(allocator, clientLocation).build()); + } + + @Override + @Test + public void testSimplePreparedStatementResultsWithParameterBinding() throws Exception { + try (PreparedStatement prepare = sqlClient.prepare("SELECT * FROM intTable WHERE id = ?")) { + final Schema parameterSchema = prepare.getParameterSchema(); + try (final VectorSchemaRoot insertRoot = VectorSchemaRoot.create(parameterSchema, allocator)) { + insertRoot.allocateNew(); + + final IntVector valueVector = (IntVector) insertRoot.getVector(0); + valueVector.setSafe(0, 1); + insertRoot.setRowCount(1); + + prepare.setParameters(insertRoot); + final FlightInfo flightInfo = prepare.execute(); + + for (FlightEndpoint endpoint: flightInfo.getEndpoints()) { + try (FlightStream stream = sqlClient.getStream(endpoint.getTicket())) { + Assertions.assertAll( + () -> MatcherAssert.assertThat(stream.getSchema(), is(SCHEMA_INT_TABLE)), + () -> MatcherAssert.assertThat(getResults(stream), is(EXPECTED_RESULTS_FOR_PARAMETER_BINDING)) + ); + } + } + } + } + } +} From 7f0c4070dd723b2f7e1967d7f7f2cccf6fb256b7 Mon Sep 17 00:00:00 2001 From: Curt Hagenlocher Date: Mon, 3 Jun 2024 13:16:05 -0700 Subject: [PATCH 43/93] GH-41397: [C#] Downgrade macOS test runner to avoid infrastructure bug (#41934) ### What changes are included in this PR? Downgrades the macOS test image for C# to use an older operating system. This works around https://github.com/pythonnet/pythonnet/issues/2396. ### Are these changes tested? Yes. ### Are there any user-facing changes? No. * GitHub Issue: #41397 Authored-by: Curt Hagenlocher Signed-off-by: Sutou Kouhei --- .github/workflows/csharp.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml index 7ae3606a44812..e4db9f482e206 100644 --- a/.github/workflows/csharp.yml +++ b/.github/workflows/csharp.yml @@ -94,8 +94,8 @@ jobs: run: ci/scripts/csharp_test.sh $(pwd) macos: - name: ARM64 macOS 14 C# ${{ matrix.dotnet }} - runs-on: macos-latest + name: AMD64 macOS 13 C# ${{ matrix.dotnet }} + runs-on: macos-13 # Pending https://github.com/pythonnet/pythonnet/issues/2396 if: ${{ !contains(github.event.pull_request.title, 'WIP') }} timeout-minutes: 15 strategy: From 2b1593d78f915b1d5e12a83ba759ed95124dd300 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Jun 2024 08:32:16 +0900 Subject: [PATCH 44/93] MINOR: [Go] Bump github.com/hamba/avro/v2 from 2.22.0 to 2.22.1 in /go (#41937) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/hamba/avro/v2](https://github.com/hamba/avro) from 2.22.0 to 2.22.1.
Release notes

Sourced from github.com/hamba/avro/v2's releases.

v2.22.1

What's Changed

New Contributors

Full Changelog: https://github.com/hamba/avro/compare/v2.22.0...v2.22.1

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/hamba/avro/v2&package-manager=go_modules&previous-version=2.22.0&new-version=2.22.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- go/go.mod | 2 +- go/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/go.mod b/go/go.mod index 9c70544539b16..b6a3ed207c6ad 100644 --- a/go/go.mod +++ b/go/go.mod @@ -47,7 +47,7 @@ require ( require ( github.com/google/uuid v1.6.0 - github.com/hamba/avro/v2 v2.22.0 + github.com/hamba/avro/v2 v2.22.1 github.com/substrait-io/substrait-go v0.4.2 github.com/tidwall/sjson v1.2.5 ) diff --git a/go/go.sum b/go/go.sum index 9e11041c333ac..79350f4a1cf27 100644 --- a/go/go.sum +++ b/go/go.sum @@ -43,8 +43,8 @@ github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbu github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/hamba/avro/v2 v2.22.0 h1:IaBMFv5xmjo38f0oaP9jZiJFXg+lmHPPg7d9YotMnPg= -github.com/hamba/avro/v2 v2.22.0/go.mod h1:HOeTrE3kvWnBAgsufqhAzDDV5gvS0QXs65Z6BHfGgbg= +github.com/hamba/avro/v2 v2.22.1 h1:q1rAbfJsrbMaZPDLQvwUQMfQzp6H+hGXvckmU/lXemk= +github.com/hamba/avro/v2 v2.22.1/go.mod h1:HOeTrE3kvWnBAgsufqhAzDDV5gvS0QXs65Z6BHfGgbg= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= From 9b59157a443dcfd7bf6f7db53a1cad02ff04645f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Jun 2024 08:32:36 +0900 Subject: [PATCH 45/93] MINOR: [Java] Bump org.apache.maven.plugins:maven-gpg-plugin from 3.2.2 to 3.2.4 in /java (#41939) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.apache.maven.plugins:maven-gpg-plugin](https://github.com/apache/maven-gpg-plugin) from 3.2.2 to 3.2.4.
Release notes

Sourced from org.apache.maven.plugins:maven-gpg-plugin's releases.

3.2.4

Release Notes - Maven GPG Plugin - Version 3.2.4

📦 Dependency updates

3.2.3

Release Notes - Maven GPG Plugin - Version 3.2.3

... (truncated)

Commits
  • 789149e [maven-release-plugin] prepare release maven-gpg-plugin-3.2.4
  • 893aedc [MGPG-125] Fix "bestPractices" (#95)
  • b6f0324 [MGPG-126] Bump commons-io:commons-io from 2.16.0 to 2.16.1 (#94)
  • 3c5878b [maven-release-plugin] prepare for next development iteration
  • 89b91a4 [maven-release-plugin] prepare release maven-gpg-plugin-3.2.3
  • fc2efa3 [MGPG-123][MGPG-124] Dependency upgrades (#93)
  • 50222d3 [MGPG-120] New mojo sign-deployed (#88)
  • a6c3a09 [MGPG-122] Bump org.apache.maven.plugins:maven-invoker-plugin from 3.6.0 to 3...
  • 78f5e37 [MGPG-121] Return the workaround for pseudo security (#90)
  • 582df74 [MGPG-117] Improve passphrase handling (#86)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.maven.plugins:maven-gpg-plugin&package-manager=maven&previous-version=3.2.2&new-version=3.2.4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- java/gandiva/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/gandiva/pom.xml b/java/gandiva/pom.xml index 26a28d55d238e..a87f26028ba86 100644 --- a/java/gandiva/pom.xml +++ b/java/gandiva/pom.xml @@ -131,7 +131,7 @@ org.apache.maven.plugins maven-gpg-plugin - 3.2.2 + 3.2.4 sign-artifacts From 9ab28c68263d6e059dd88908aaf8d2a5cf9b5eeb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Jun 2024 08:32:55 +0900 Subject: [PATCH 46/93] MINOR: [Java] Bump dep.slf4j.version from 2.0.12 to 2.0.13 in /java (#41940) Bumps `dep.slf4j.version` from 2.0.12 to 2.0.13. Updates `org.slf4j:slf4j-api` from 2.0.12 to 2.0.13 Updates `org.slf4j:slf4j-jdk14` from 2.0.12 to 2.0.13 Updates `org.slf4j:jul-to-slf4j` from 2.0.12 to 2.0.13 Updates `org.slf4j:jcl-over-slf4j` from 2.0.12 to 2.0.13 Updates `org.slf4j:log4j-over-slf4j` from 2.0.12 to 2.0.13 Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- java/maven/pom.xml | 2 +- java/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/java/maven/pom.xml b/java/maven/pom.xml index f290ded2e2913..470e198caebc1 100644 --- a/java/maven/pom.xml +++ b/java/maven/pom.xml @@ -272,7 +272,7 @@ org.slf4j jcl-over-slf4j - 2.0.12 + 2.0.13 diff --git a/java/pom.xml b/java/pom.xml index 289810daba3ac..e932c749bd832 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -81,7 +81,7 @@ ${project.build.directory}/generated-sources 1.9.0 5.10.2 - 2.0.12 + 2.0.13 33.0.0-jre 4.1.108.Final 1.63.0 From fd54260f2982c898fc8fc91752278446e2fdc56a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Jun 2024 08:33:16 +0900 Subject: [PATCH 47/93] MINOR: [Java] Bump org.apache.maven.plugins:maven-install-plugin from 3.1.1 to 3.1.2 in /java (#41941) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [org.apache.maven.plugins:maven-install-plugin](https://github.com/apache/maven-install-plugin) from 3.1.1 to 3.1.2.
Release notes

Sourced from org.apache.maven.plugins:maven-install-plugin's releases.

3.1.2

Release Notes - Maven Install Plugin - Version 3.1.2

What's Changed

New Contributors

Full Changelog: https://github.com/apache/maven-install-plugin/compare/maven-install-plugin-3.1.1...maven-install-plugin-3.1.2

Commits
  • e1494c1 [maven-release-plugin] prepare release maven-install-plugin-3.1.2
  • 6712ae8 Remove workaround, handle it centrally
  • 1fa847f Bump workflow
  • bdc93d5 Quickfix: move to zulu
  • 7b9bf51 [MINSTALL-193] Parent 42, min Maven 3.6.3 (#64)
  • e914367 [MINSTALL-195] Include artifactId in InstallMojo#processProject messages
  • 3ebb448 [MINSTALL-197] Update to parent 41, cleanup (#61)
  • 30d2b53 [MINSTALL-192] - Code cleanups
  • 429ad5b [MNG-6829] Replace StringUtils#isEmpty(String) & #isNotEmpty(String) (#58)
  • f6377c4 configure notifications
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.maven.plugins:maven-install-plugin&package-manager=maven&previous-version=3.1.1&new-version=3.1.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- java/maven/module-info-compiler-maven-plugin/pom.xml | 2 +- java/performance/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/java/maven/module-info-compiler-maven-plugin/pom.xml b/java/maven/module-info-compiler-maven-plugin/pom.xml index 9c1e8fe058110..57ba7933ea1c6 100644 --- a/java/maven/module-info-compiler-maven-plugin/pom.xml +++ b/java/maven/module-info-compiler-maven-plugin/pom.xml @@ -87,7 +87,7 @@ maven-install-plugin - 3.1.1 + 3.1.2 maven-deploy-plugin diff --git a/java/performance/pom.xml b/java/performance/pom.xml index 765b6a58cd8f0..f01e8d9a4e0e4 100644 --- a/java/performance/pom.xml +++ b/java/performance/pom.xml @@ -95,7 +95,7 @@ maven-install-plugin - 3.1.1 + 3.1.2 maven-jar-plugin From b08c8be3d63718b1ffb1699ba21b34d7c8c831c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Jun 2024 08:39:19 +0900 Subject: [PATCH 48/93] MINOR: [Java] Bump com.google.guava:guava-bom from 33.0.0-jre to 33.2.1-jre in /java (#41943) Bumps [com.google.guava:guava-bom](https://github.com/google/guava) from 33.0.0-jre to 33.2.1-jre.
Release notes

Sourced from com.google.guava:guava-bom's releases.

33.2.1

<dependency>
  <groupId>com.google.guava</groupId>
  <artifactId>guava</artifactId>
  <version>33.2.1-jre</version>
  <!-- or, for Android: -->
  <version>33.2.1-android</version>
</dependency>

Jar files

Guava requires one runtime dependency, which you can download here:

Javadoc

JDiff

Changelog

  • net: Changed InetAddress-String conversion methods to preserve the IPv6 scope ID if present. The scope ID can be necessary for IPv6-capable devices with multiple network interfaces. However, preserving it can also lead to problems for callers that rely on the returned values not to include the scope ID:
    • Callers might compensate for the old behavior of the methods by appending the scope ID to a returned string themselves. If so, you can update your code to stop doing so at the same time as you upgrade Guava. Of, if your code might run against multiple versions of Guava, you can check whether Guava has included a scope ID before you add one yourself.
    • Callers might pass the returned string to another system that does not understand scope IDs. If so, you can strip the scope ID off, whether by truncating the string form at a % character (leaving behind any trailing ] character in the case of forUriString) or by replacing the returned InetAddress with a new instance constructed by calling InetAddress.getByAddress(addr).
    • java.net.InetAddress validates any provided scope ID against the interfaces available on the machine. As a result, methods in InetAddresses may now fail if the scope ID fails validation.
      • Notable cases in which this may happen include:
        • if the code runs in an Android app without networking permission
        • if code passes InetAddress instances or strings across devices
      • If this is not the behavior that you want, then you can strip off the scope ID from the input string before passing it to Guava, as discussed above. (3f61870ac6)

33.2.0

Android users: Please test recent Guava versions

If you know of Guava Android users who have not yet upgraded to at least release 33.0.0, please encourage them to upgrade, preferably to today's release, 33.2.0. These releases have begun adding Java 8+ APIs to guava-android. While we don't anticipate problems, we do anticipate that any unexpected problems could force a disruptive rollback. To minimize any disruption, we'd like to catch any such problems early.

Please let us know of any problems you encounter.

Maven

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.google.guava:guava-bom&package-manager=maven&previous-version=33.0.0-jre&new-version=33.2.1-jre)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- java/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/pom.xml b/java/pom.xml index e932c749bd832..9be9d431d4776 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -82,7 +82,7 @@ 1.9.0 5.10.2 2.0.13 - 33.0.0-jre + 33.2.1-jre 4.1.108.Final 1.63.0 3.23.1 From 2c2c6c505d11d0db0d41013ccdf50527c9a8ed18 Mon Sep 17 00:00:00 2001 From: Dominik Moritz Date: Mon, 3 Jun 2024 22:00:52 -0400 Subject: [PATCH 49/93] GH-41905: [JS] Update dependencies (#41906) --- js/bin/integration.ts | 2 +- js/package.json | 24 +- js/test/unit/builders/builder-tests.ts | 2 +- js/test/unit/builders/utils.ts | 2 +- js/test/unit/ipc/writer/streams-dom-tests.ts | 4 +- js/test/unit/ipc/writer/streams-node-tests.ts | 4 +- js/test/unit/table/assign-tests.ts | 2 +- js/yarn.lock | 846 ++++++++---------- 8 files changed, 414 insertions(+), 472 deletions(-) diff --git a/js/bin/integration.ts b/js/bin/integration.ts index f73388cc85cf0..d41ce08aa16b6 100755 --- a/js/bin/integration.ts +++ b/js/bin/integration.ts @@ -20,7 +20,7 @@ import * as fs from 'node:fs'; import * as Path from 'node:path'; import { glob } from 'glob'; -import { zip } from 'ix/iterable/zip.js'; +import { zip } from 'ix/iterable/zip'; import commandLineArgs from 'command-line-args'; // @ts-ignore import { parse as bignumJSONParse } from 'json-bignum'; diff --git a/js/package.json b/js/package.json index 7ed0daddfada0..9e61d94dc3b7b 100644 --- a/js/package.json +++ b/js/package.json @@ -52,10 +52,10 @@ "jest.config.js" ], "dependencies": { - "@swc/helpers": "^0.5.10", + "@swc/helpers": "^0.5.11", "@types/command-line-args": "^5.2.3", "@types/command-line-usage": "^5.0.4", - "@types/node": "^20.12.7", + "@types/node": "^20.13.0", "command-line-args": "^5.2.1", "command-line-usage": "^7.0.1", "flatbuffers": "^24.3.25", @@ -67,26 +67,26 @@ "@rollup/plugin-alias": "5.1.0", "@rollup/plugin-node-resolve": "15.2.3", "@rollup/stream": "3.0.1", - "@swc/core": "1.4.17", + "@swc/core": "1.5.24", "@types/benchmark": "2.1.5", "@types/glob": "8.1.0", "@types/jest": "29.5.12", "@types/multistream": "4.1.3", - "@typescript-eslint/eslint-plugin": "7.8.0", - "@typescript-eslint/parser": "7.8.0", + "@typescript-eslint/eslint-plugin": "7.11.0", + "@typescript-eslint/parser": "7.11.0", "async-done": "2.0.0", "benny": "3.7.1", "cross-env": "7.0.3", "del": "7.1.0", "del-cli": "5.1.0", - "esbuild": "0.20.2", + "esbuild": "0.21.4", "esbuild-plugin-alias": "0.2.1", "eslint": "8.57.0", - "eslint-plugin-jest": "28.4.0", + "eslint-plugin-jest": "28.5.0", "eslint-plugin-unicorn": "52.0.0", "esm": "https://github.com/jsg2021/esm/releases/download/v3.x.x-pr883/esm-3.x.x-pr883.tgz", "gulp": "4.0.2", - "glob": "10.3.12", + "glob": "10.4.1", "google-closure-compiler": "20240317.0.0", "gulp-esbuild": "0.12.0", "gulp-json-transform": "0.5.0", @@ -96,16 +96,16 @@ "gulp-terser": "2.1.0", "gulp-typescript": "5.0.1", "gulp-vinyl-size": "1.1.4", - "ix": "5.0.0", + "ix": "6.0.0", "jest": "29.7.0", - "jest-silent-reporter": "0.5.0", + "jest-silent-reporter": "0.6.0", "memfs": "4.9.2", "mkdirp": "3.0.1", "multistream": "4.1.0", "regenerator-runtime": "0.14.1", - "rollup": "4.17.2", + "rollup": "4.18.0", "rxjs": "7.8.1", - "ts-jest": "29.1.2", + "ts-jest": "29.1.4", "ts-node": "10.9.2", "typedoc": "0.25.13", "typescript": "5.4.5", diff --git a/js/test/unit/builders/builder-tests.ts b/js/test/unit/builders/builder-tests.ts index 4d1be9b225b08..c9174023f6dae 100644 --- a/js/test/unit/builders/builder-tests.ts +++ b/js/test/unit/builders/builder-tests.ts @@ -18,7 +18,7 @@ import 'web-streams-polyfill'; import '../../jest-extensions.js'; -import { from, fromDOMStream, toArray } from 'ix/asynciterable'; +import { from, fromDOMStream, toArray } from 'ix/Ix.asynciterable'; import { fromNodeStream } from 'ix/asynciterable/fromnodestream'; import { validateVector } from './utils.js'; diff --git a/js/test/unit/builders/utils.ts b/js/test/unit/builders/utils.ts index 1d0707a6ca5d9..7cc0a075d84d4 100644 --- a/js/test/unit/builders/utils.ts +++ b/js/test/unit/builders/utils.ts @@ -17,7 +17,7 @@ import 'web-streams-polyfill'; -import { from, fromDOMStream, toArray } from 'ix/asynciterable'; +import { from, fromDOMStream, toArray } from 'ix/Ix.asynciterable'; import { fromNodeStream } from 'ix/asynciterable/fromnodestream'; import 'ix/Ix.node'; diff --git a/js/test/unit/ipc/writer/streams-dom-tests.ts b/js/test/unit/ipc/writer/streams-dom-tests.ts index dc792c9cf82be..2040e89a48802 100644 --- a/js/test/unit/ipc/writer/streams-dom-tests.ts +++ b/js/test/unit/ipc/writer/streams-dom-tests.ts @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -import { as, from } from 'ix/asynciterable'; -import { flatMap, tap } from 'ix/asynciterable/operators'; +import { as, from } from 'ix/Ix.asynciterable'; +import { flatMap, tap } from 'ix/Ix.asynciterable.operators'; import { generateRandomTables } from '../../../data/tables.js'; import { diff --git a/js/test/unit/ipc/writer/streams-node-tests.ts b/js/test/unit/ipc/writer/streams-node-tests.ts index 1f4c9c7a02cfb..afcb6deb1e053 100644 --- a/js/test/unit/ipc/writer/streams-node-tests.ts +++ b/js/test/unit/ipc/writer/streams-node-tests.ts @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -import { as, from } from 'ix/asynciterable'; -import { flatMap, tap } from 'ix/asynciterable/operators'; +import { as, from } from 'ix/Ix.asynciterable'; +import { flatMap, tap } from 'ix/Ix.asynciterable.operators'; import 'ix/Ix.node'; import { generateRandomTables } from '../../../data/tables.js'; diff --git a/js/test/unit/table/assign-tests.ts b/js/test/unit/table/assign-tests.ts index f2a5ff4f37cac..8e1cdfec41cbf 100644 --- a/js/test/unit/table/assign-tests.ts +++ b/js/test/unit/table/assign-tests.ts @@ -17,7 +17,7 @@ /* eslint-disable jest/no-standalone-expect */ -import { zip } from 'ix/iterable'; +import { zip } from 'ix/Ix.iterable'; import '../../jest-extensions.js'; import * as generate from '../../generate-test-data.js'; diff --git a/js/yarn.lock b/js/yarn.lock index eb7ed33520f0a..d5527097340d9 100644 --- a/js/yarn.lock +++ b/js/yarn.lock @@ -362,230 +362,230 @@ resolved "https://registry.yarnpkg.com/@esbuild/aix-ppc64/-/aix-ppc64-0.19.12.tgz#d1bc06aedb6936b3b6d313bf809a5a40387d2b7f" integrity sha512-bmoCYyWdEL3wDQIVbcyzRyeKLgk2WtWLTWz1ZIAZF/EGbNOwSA6ew3PftJ1PqMiOOGu0OyFMzG53L0zqIpPeNA== -"@esbuild/aix-ppc64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/aix-ppc64/-/aix-ppc64-0.20.2.tgz#a70f4ac11c6a1dfc18b8bbb13284155d933b9537" - integrity sha512-D+EBOJHXdNZcLJRBkhENNG8Wji2kgc9AZ9KiPr1JuZjsNtyHzrsfLRrY0tk2H2aoFu6RANO1y1iPPUCDYWkb5g== +"@esbuild/aix-ppc64@0.21.4": + version "0.21.4" + resolved "https://registry.yarnpkg.com/@esbuild/aix-ppc64/-/aix-ppc64-0.21.4.tgz#f83eb142df3ca7b49531c1ed680b81e484316508" + integrity sha512-Zrm+B33R4LWPLjDEVnEqt2+SLTATlru1q/xYKVn8oVTbiRBGmK2VIMoIYGJDGyftnGaC788IuzGFAlb7IQ0Y8A== "@esbuild/android-arm64@0.19.12": version "0.19.12" resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.19.12.tgz#7ad65a36cfdb7e0d429c353e00f680d737c2aed4" integrity sha512-P0UVNGIienjZv3f5zq0DP3Nt2IE/3plFzuaS96vihvD0Hd6H/q4WXUGpCxD/E8YrSXfNyRPbpTq+T8ZQioSuPA== -"@esbuild/android-arm64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.20.2.tgz#db1c9202a5bc92ea04c7b6840f1bbe09ebf9e6b9" - integrity sha512-mRzjLacRtl/tWU0SvD8lUEwb61yP9cqQo6noDZP/O8VkwafSYwZ4yWy24kan8jE/IMERpYncRt2dw438LP3Xmg== +"@esbuild/android-arm64@0.21.4": + version "0.21.4" + resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.21.4.tgz#dd328039daccd6033b2d1e536c054914bfc92287" + integrity sha512-fYFnz+ObClJ3dNiITySBUx+oNalYUT18/AryMxfovLkYWbutXsct3Wz2ZWAcGGppp+RVVX5FiXeLYGi97umisA== "@esbuild/android-arm@0.19.12": version "0.19.12" resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.19.12.tgz#b0c26536f37776162ca8bde25e42040c203f2824" integrity sha512-qg/Lj1mu3CdQlDEEiWrlC4eaPZ1KztwGJ9B6J+/6G+/4ewxJg7gqj8eVYWvao1bXrqGiW2rsBZFSX3q2lcW05w== -"@esbuild/android-arm@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.20.2.tgz#3b488c49aee9d491c2c8f98a909b785870d6e995" - integrity sha512-t98Ra6pw2VaDhqNWO2Oph2LXbz/EJcnLmKLGBJwEwXX/JAN83Fym1rU8l0JUWK6HkIbWONCSSatf4sf2NBRx/w== +"@esbuild/android-arm@0.21.4": + version "0.21.4" + resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.21.4.tgz#76767a989720a97b206ea14c52af6e4589e48b0d" + integrity sha512-E7H/yTd8kGQfY4z9t3nRPk/hrhaCajfA3YSQSBrst8B+3uTcgsi8N+ZWYCaeIDsiVs6m65JPCaQN/DxBRclF3A== "@esbuild/android-x64@0.19.12": version "0.19.12" resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.19.12.tgz#cb13e2211282012194d89bf3bfe7721273473b3d" integrity sha512-3k7ZoUW6Q6YqhdhIaq/WZ7HwBpnFBlW905Fa4s4qWJyiNOgT1dOqDiVAQFwBH7gBRZr17gLrlFCRzF6jFh7Kew== -"@esbuild/android-x64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.20.2.tgz#3b1628029e5576249d2b2d766696e50768449f98" - integrity sha512-btzExgV+/lMGDDa194CcUQm53ncxzeBrWJcncOBxuC6ndBkKxnHdFJn86mCIgTELsooUmwUm9FkhSp5HYu00Rg== +"@esbuild/android-x64@0.21.4": + version "0.21.4" + resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.21.4.tgz#14a8ae3c35702d882086efb5a8f8d7b0038d8d35" + integrity sha512-mDqmlge3hFbEPbCWxp4fM6hqq7aZfLEHZAKGP9viq9wMUBVQx202aDIfc3l+d2cKhUJM741VrCXEzRFhPDKH3Q== "@esbuild/darwin-arm64@0.19.12": version "0.19.12" resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.19.12.tgz#cbee41e988020d4b516e9d9e44dd29200996275e" integrity sha512-B6IeSgZgtEzGC42jsI+YYu9Z3HKRxp8ZT3cqhvliEHovq8HSX2YX8lNocDn79gCKJXOSaEot9MVYky7AKjCs8g== -"@esbuild/darwin-arm64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.20.2.tgz#6e8517a045ddd86ae30c6608c8475ebc0c4000bb" - integrity sha512-4J6IRT+10J3aJH3l1yzEg9y3wkTDgDk7TSDFX+wKFiWjqWp/iCfLIYzGyasx9l0SAFPT1HwSCR+0w/h1ES/MjA== +"@esbuild/darwin-arm64@0.21.4": + version "0.21.4" + resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.21.4.tgz#7e735046005e4c12e9139e0bdd1fa6a754430d57" + integrity sha512-72eaIrDZDSiWqpmCzVaBD58c8ea8cw/U0fq/PPOTqE3c53D0xVMRt2ooIABZ6/wj99Y+h4ksT/+I+srCDLU9TA== "@esbuild/darwin-x64@0.19.12": version "0.19.12" resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.19.12.tgz#e37d9633246d52aecf491ee916ece709f9d5f4cd" integrity sha512-hKoVkKzFiToTgn+41qGhsUJXFlIjxI/jSYeZf3ugemDYZldIXIxhvwN6erJGlX4t5h417iFuheZ7l+YVn05N3A== -"@esbuild/darwin-x64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.20.2.tgz#90ed098e1f9dd8a9381695b207e1cff45540a0d0" - integrity sha512-tBcXp9KNphnNH0dfhv8KYkZhjc+H3XBkF5DKtswJblV7KlT9EI2+jeA8DgBjp908WEuYll6pF+UStUCfEpdysA== +"@esbuild/darwin-x64@0.21.4": + version "0.21.4" + resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.21.4.tgz#db623553547a5fe3502a63aa88306e9023178482" + integrity sha512-uBsuwRMehGmw1JC7Vecu/upOjTsMhgahmDkWhGLWxIgUn2x/Y4tIwUZngsmVb6XyPSTXJYS4YiASKPcm9Zitag== "@esbuild/freebsd-arm64@0.19.12": version "0.19.12" resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.19.12.tgz#1ee4d8b682ed363b08af74d1ea2b2b4dbba76487" integrity sha512-4aRvFIXmwAcDBw9AueDQ2YnGmz5L6obe5kmPT8Vd+/+x/JMVKCgdcRwH6APrbpNXsPz+K653Qg8HB/oXvXVukA== -"@esbuild/freebsd-arm64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.20.2.tgz#d71502d1ee89a1130327e890364666c760a2a911" - integrity sha512-d3qI41G4SuLiCGCFGUrKsSeTXyWG6yem1KcGZVS+3FYlYhtNoNgYrWcvkOoaqMhwXSMrZRl69ArHsGJ9mYdbbw== +"@esbuild/freebsd-arm64@0.21.4": + version "0.21.4" + resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.4.tgz#91cbad647c079bf932086fbd4749d7f563df67b8" + integrity sha512-8JfuSC6YMSAEIZIWNL3GtdUT5NhUA/CMUCpZdDRolUXNAXEE/Vbpe6qlGLpfThtY5NwXq8Hi4nJy4YfPh+TwAg== "@esbuild/freebsd-x64@0.19.12": version "0.19.12" resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.19.12.tgz#37a693553d42ff77cd7126764b535fb6cc28a11c" integrity sha512-EYoXZ4d8xtBoVN7CEwWY2IN4ho76xjYXqSXMNccFSx2lgqOG/1TBPW0yPx1bJZk94qu3tX0fycJeeQsKovA8gg== -"@esbuild/freebsd-x64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.20.2.tgz#aa5ea58d9c1dd9af688b8b6f63ef0d3d60cea53c" - integrity sha512-d+DipyvHRuqEeM5zDivKV1KuXn9WeRX6vqSqIDgwIfPQtwMP4jaDsQsDncjTDDsExT4lR/91OLjRo8bmC1e+Cw== +"@esbuild/freebsd-x64@0.21.4": + version "0.21.4" + resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.21.4.tgz#723299b9859ccbe5532fecbadba3ac33019ba8e8" + integrity sha512-8d9y9eQhxv4ef7JmXny7591P/PYsDFc4+STaxC1GBv0tMyCdyWfXu2jBuqRsyhY8uL2HU8uPyscgE2KxCY9imQ== "@esbuild/linux-arm64@0.19.12": version "0.19.12" resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.19.12.tgz#be9b145985ec6c57470e0e051d887b09dddb2d4b" integrity sha512-EoTjyYyLuVPfdPLsGVVVC8a0p1BFFvtpQDB/YLEhaXyf/5bczaGeN15QkR+O4S5LeJ92Tqotve7i1jn35qwvdA== -"@esbuild/linux-arm64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.20.2.tgz#055b63725df678379b0f6db9d0fa85463755b2e5" - integrity sha512-9pb6rBjGvTFNira2FLIWqDk/uaf42sSyLE8j1rnUpuzsODBq7FvpwHYZxQ/It/8b+QOS1RYfqgGFNLRI+qlq2A== +"@esbuild/linux-arm64@0.21.4": + version "0.21.4" + resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.21.4.tgz#531743f861e1ef6e50b874d6c784cda37aa5e685" + integrity sha512-/GLD2orjNU50v9PcxNpYZi+y8dJ7e7/LhQukN3S4jNDXCKkyyiyAz9zDw3siZ7Eh1tRcnCHAo/WcqKMzmi4eMQ== "@esbuild/linux-arm@0.19.12": version "0.19.12" resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.19.12.tgz#207ecd982a8db95f7b5279207d0ff2331acf5eef" integrity sha512-J5jPms//KhSNv+LO1S1TX1UWp1ucM6N6XuL6ITdKWElCu8wXP72l9MM0zDTzzeikVyqFE6U8YAV9/tFyj0ti+w== -"@esbuild/linux-arm@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.20.2.tgz#76b3b98cb1f87936fbc37f073efabad49dcd889c" - integrity sha512-VhLPeR8HTMPccbuWWcEUD1Az68TqaTYyj6nfE4QByZIQEQVWBB8vup8PpR7y1QHL3CpcF6xd5WVBU/+SBEvGTg== +"@esbuild/linux-arm@0.21.4": + version "0.21.4" + resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.21.4.tgz#1144b5654764960dd97d90ddf0893a9afc63ad91" + integrity sha512-2rqFFefpYmpMs+FWjkzSgXg5vViocqpq5a1PSRgT0AvSgxoXmGF17qfGAzKedg6wAwyM7UltrKVo9kxaJLMF/g== "@esbuild/linux-ia32@0.19.12": version "0.19.12" resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.19.12.tgz#d0d86b5ca1562523dc284a6723293a52d5860601" integrity sha512-Thsa42rrP1+UIGaWz47uydHSBOgTUnwBwNq59khgIwktK6x60Hivfbux9iNR0eHCHzOLjLMLfUMLCypBkZXMHA== -"@esbuild/linux-ia32@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.20.2.tgz#c0e5e787c285264e5dfc7a79f04b8b4eefdad7fa" - integrity sha512-o10utieEkNPFDZFQm9CoP7Tvb33UutoJqg3qKf1PWVeeJhJw0Q347PxMvBgVVFgouYLGIhFYG0UGdBumROyiig== +"@esbuild/linux-ia32@0.21.4": + version "0.21.4" + resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.21.4.tgz#c81b6f2ed3308d3b75ccefb5ac63bc4cf3a9d2e9" + integrity sha512-pNftBl7m/tFG3t2m/tSjuYeWIffzwAZT9m08+9DPLizxVOsUl8DdFzn9HvJrTQwe3wvJnwTdl92AonY36w/25g== "@esbuild/linux-loong64@0.19.12": version "0.19.12" resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.19.12.tgz#9a37f87fec4b8408e682b528391fa22afd952299" integrity sha512-LiXdXA0s3IqRRjm6rV6XaWATScKAXjI4R4LoDlvO7+yQqFdlr1Bax62sRwkVvRIrwXxvtYEHHI4dm50jAXkuAA== -"@esbuild/linux-loong64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.20.2.tgz#a6184e62bd7cdc63e0c0448b83801001653219c5" - integrity sha512-PR7sp6R/UC4CFVomVINKJ80pMFlfDfMQMYynX7t1tNTeivQ6XdX5r2XovMmha/VjR1YN/HgHWsVcTRIMkymrgQ== +"@esbuild/linux-loong64@0.21.4": + version "0.21.4" + resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.21.4.tgz#87b6af7cd0f2551653955fc2dc465b7f4464af0a" + integrity sha512-cSD2gzCK5LuVX+hszzXQzlWya6c7hilO71L9h4KHwqI4qeqZ57bAtkgcC2YioXjsbfAv4lPn3qe3b00Zt+jIfQ== "@esbuild/linux-mips64el@0.19.12": version "0.19.12" resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.19.12.tgz#4ddebd4e6eeba20b509d8e74c8e30d8ace0b89ec" integrity sha512-fEnAuj5VGTanfJ07ff0gOA6IPsvrVHLVb6Lyd1g2/ed67oU1eFzL0r9WL7ZzscD+/N6i3dWumGE1Un4f7Amf+w== -"@esbuild/linux-mips64el@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.20.2.tgz#d08e39ce86f45ef8fc88549d29c62b8acf5649aa" - integrity sha512-4BlTqeutE/KnOiTG5Y6Sb/Hw6hsBOZapOVF6njAESHInhlQAghVVZL1ZpIctBOoTFbQyGW+LsVYZ8lSSB3wkjA== +"@esbuild/linux-mips64el@0.21.4": + version "0.21.4" + resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.21.4.tgz#fec73cd39490a0c45d052bef03e011a0ad366c06" + integrity sha512-qtzAd3BJh7UdbiXCrg6npWLYU0YpufsV9XlufKhMhYMJGJCdfX/G6+PNd0+v877X1JG5VmjBLUiFB0o8EUSicA== "@esbuild/linux-ppc64@0.19.12": version "0.19.12" resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.19.12.tgz#adb67dadb73656849f63cd522f5ecb351dd8dee8" integrity sha512-nYJA2/QPimDQOh1rKWedNOe3Gfc8PabU7HT3iXWtNUbRzXS9+vgB0Fjaqr//XNbd82mCxHzik2qotuI89cfixg== -"@esbuild/linux-ppc64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.20.2.tgz#8d252f0b7756ffd6d1cbde5ea67ff8fd20437f20" - integrity sha512-rD3KsaDprDcfajSKdn25ooz5J5/fWBylaaXkuotBDGnMnDP1Uv5DLAN/45qfnf3JDYyJv/ytGHQaziHUdyzaAg== +"@esbuild/linux-ppc64@0.21.4": + version "0.21.4" + resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.21.4.tgz#ea3b5e13b0fc8666bd4c6f7ea58bd1830f3e6e78" + integrity sha512-yB8AYzOTaL0D5+2a4xEy7OVvbcypvDR05MsB/VVPVA7nL4hc5w5Dyd/ddnayStDgJE59fAgNEOdLhBxjfx5+dg== "@esbuild/linux-riscv64@0.19.12": version "0.19.12" resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.19.12.tgz#11bc0698bf0a2abf8727f1c7ace2112612c15adf" integrity sha512-2MueBrlPQCw5dVJJpQdUYgeqIzDQgw3QtiAHUC4RBz9FXPrskyyU3VI1hw7C0BSKB9OduwSJ79FTCqtGMWqJHg== -"@esbuild/linux-riscv64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.20.2.tgz#19f6dcdb14409dae607f66ca1181dd4e9db81300" - integrity sha512-snwmBKacKmwTMmhLlz/3aH1Q9T8v45bKYGE3j26TsaOVtjIag4wLfWSiZykXzXuE1kbCE+zJRmwp+ZbIHinnVg== +"@esbuild/linux-riscv64@0.21.4": + version "0.21.4" + resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.21.4.tgz#80d406f653fc6b193edaeb55ac88d4ac22c8f155" + integrity sha512-Y5AgOuVzPjQdgU59ramLoqSSiXddu7F3F+LI5hYy/d1UHN7K5oLzYBDZe23QmQJ9PIVUXwOdKJ/jZahPdxzm9w== "@esbuild/linux-s390x@0.19.12": version "0.19.12" resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.19.12.tgz#e86fb8ffba7c5c92ba91fc3b27ed5a70196c3cc8" integrity sha512-+Pil1Nv3Umes4m3AZKqA2anfhJiVmNCYkPchwFJNEJN5QxmTs1uzyy4TvmDrCRNT2ApwSari7ZIgrPeUx4UZDg== -"@esbuild/linux-s390x@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.20.2.tgz#3c830c90f1a5d7dd1473d5595ea4ebb920988685" - integrity sha512-wcWISOobRWNm3cezm5HOZcYz1sKoHLd8VL1dl309DiixxVFoFe/o8HnwuIwn6sXre88Nwj+VwZUvJf4AFxkyrQ== +"@esbuild/linux-s390x@0.21.4": + version "0.21.4" + resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.21.4.tgz#9cbd26854b5b12cf22fb54c96cd1adffaf6ace6f" + integrity sha512-Iqc/l/FFwtt8FoTK9riYv9zQNms7B8u+vAI/rxKuN10HgQIXaPzKZc479lZ0x6+vKVQbu55GdpYpeNWzjOhgbA== "@esbuild/linux-x64@0.19.12": version "0.19.12" resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.19.12.tgz#5f37cfdc705aea687dfe5dfbec086a05acfe9c78" integrity sha512-B71g1QpxfwBvNrfyJdVDexenDIt1CiDN1TIXLbhOw0KhJzE78KIFGX6OJ9MrtC0oOqMWf+0xop4qEU8JrJTwCg== -"@esbuild/linux-x64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.20.2.tgz#86eca35203afc0d9de0694c64ec0ab0a378f6fff" - integrity sha512-1MdwI6OOTsfQfek8sLwgyjOXAu+wKhLEoaOLTjbijk6E2WONYpH9ZU2mNtR+lZ2B4uwr+usqGuVfFT9tMtGvGw== +"@esbuild/linux-x64@0.21.4": + version "0.21.4" + resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.21.4.tgz#44dfe1c5cad855362c830c604dba97fbb16fc114" + integrity sha512-Td9jv782UMAFsuLZINfUpoF5mZIbAj+jv1YVtE58rFtfvoKRiKSkRGQfHTgKamLVT/fO7203bHa3wU122V/Bdg== "@esbuild/netbsd-x64@0.19.12": version "0.19.12" resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.19.12.tgz#29da566a75324e0d0dd7e47519ba2f7ef168657b" integrity sha512-3ltjQ7n1owJgFbuC61Oj++XhtzmymoCihNFgT84UAmJnxJfm4sYCiSLTXZtE00VWYpPMYc+ZQmB6xbSdVh0JWA== -"@esbuild/netbsd-x64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.20.2.tgz#e771c8eb0e0f6e1877ffd4220036b98aed5915e6" - integrity sha512-K8/DhBxcVQkzYc43yJXDSyjlFeHQJBiowJ0uVL6Tor3jGQfSGHNNJcWxNbOI8v5k82prYqzPuwkzHt3J1T1iZQ== +"@esbuild/netbsd-x64@0.21.4": + version "0.21.4" + resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.21.4.tgz#89b97d823e1cc4bf8c4e5dc8f76c8d6ceb1c87f3" + integrity sha512-Awn38oSXxsPMQxaV0Ipb7W/gxZtk5Tx3+W+rAPdZkyEhQ6968r9NvtkjhnhbEgWXYbgV+JEONJ6PcdBS+nlcpA== "@esbuild/openbsd-x64@0.19.12": version "0.19.12" resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.19.12.tgz#306c0acbdb5a99c95be98bdd1d47c916e7dc3ff0" integrity sha512-RbrfTB9SWsr0kWmb9srfF+L933uMDdu9BIzdA7os2t0TXhCRjrQyCeOt6wVxr79CKD4c+p+YhCj31HBkYcXebw== -"@esbuild/openbsd-x64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.20.2.tgz#9a795ae4b4e37e674f0f4d716f3e226dd7c39baf" - integrity sha512-eMpKlV0SThJmmJgiVyN9jTPJ2VBPquf6Kt/nAoo6DgHAoN57K15ZghiHaMvqjCye/uU4X5u3YSMgVBI1h3vKrQ== +"@esbuild/openbsd-x64@0.21.4": + version "0.21.4" + resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.21.4.tgz#080715bb4981c326364320d7b56835608e2bd98d" + integrity sha512-IsUmQeCY0aU374R82fxIPu6vkOybWIMc3hVGZ3ChRwL9hA1TwY+tS0lgFWV5+F1+1ssuvvXt3HFqe8roCip8Hg== "@esbuild/sunos-x64@0.19.12": version "0.19.12" resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.19.12.tgz#0933eaab9af8b9b2c930236f62aae3fc593faf30" integrity sha512-HKjJwRrW8uWtCQnQOz9qcU3mUZhTUQvi56Q8DPTLLB+DawoiQdjsYq+j+D3s9I8VFtDr+F9CjgXKKC4ss89IeA== -"@esbuild/sunos-x64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.20.2.tgz#7df23b61a497b8ac189def6e25a95673caedb03f" - integrity sha512-2UyFtRC6cXLyejf/YEld4Hajo7UHILetzE1vsRcGL3earZEW77JxrFjH4Ez2qaTiEfMgAXxfAZCm1fvM/G/o8w== +"@esbuild/sunos-x64@0.21.4": + version "0.21.4" + resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.21.4.tgz#8d838a8ac80e211536490108b72fb0091a811626" + integrity sha512-hsKhgZ4teLUaDA6FG/QIu2q0rI6I36tZVfM4DBZv3BG0mkMIdEnMbhc4xwLvLJSS22uWmaVkFkqWgIS0gPIm+A== "@esbuild/win32-arm64@0.19.12": version "0.19.12" resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.19.12.tgz#773bdbaa1971b36db2f6560088639ccd1e6773ae" integrity sha512-URgtR1dJnmGvX864pn1B2YUYNzjmXkuJOIqG2HdU62MVS4EHpU2946OZoTMnRUHklGtJdJZ33QfzdjGACXhn1A== -"@esbuild/win32-arm64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.20.2.tgz#f1ae5abf9ca052ae11c1bc806fb4c0f519bacf90" - integrity sha512-GRibxoawM9ZCnDxnP3usoUDO9vUkpAxIIZ6GQI+IlVmr5kP3zUq+l17xELTHMWTWzjxa2guPNyrpq1GWmPvcGQ== +"@esbuild/win32-arm64@0.21.4": + version "0.21.4" + resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.21.4.tgz#94afb4c2ac89b0f09791606d6d93fdab322f81c8" + integrity sha512-UUfMgMoXPoA/bvGUNfUBFLCh0gt9dxZYIx9W4rfJr7+hKe5jxxHmfOK8YSH4qsHLLN4Ck8JZ+v7Q5fIm1huErg== "@esbuild/win32-ia32@0.19.12": version "0.19.12" resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.19.12.tgz#000516cad06354cc84a73f0943a4aa690ef6fd67" integrity sha512-+ZOE6pUkMOJfmxmBZElNOx72NKpIa/HFOMGzu8fqzQJ5kgf6aTGrcJaFsNiVMH4JKpMipyK+7k0n2UXN7a8YKQ== -"@esbuild/win32-ia32@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.20.2.tgz#241fe62c34d8e8461cd708277813e1d0ba55ce23" - integrity sha512-HfLOfn9YWmkSKRQqovpnITazdtquEW8/SoHW7pWpuEeguaZI4QnCRW6b+oZTztdBnZOS2hqJ6im/D5cPzBTTlQ== +"@esbuild/win32-ia32@0.21.4": + version "0.21.4" + resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.21.4.tgz#822085cd52f2f1dd90eabb59346ffa779c0bab83" + integrity sha512-yIxbspZb5kGCAHWm8dexALQ9en1IYDfErzjSEq1KzXFniHv019VT3mNtTK7t8qdy4TwT6QYHI9sEZabONHg+aw== "@esbuild/win32-x64@0.19.12": version "0.19.12" resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.19.12.tgz#c57c8afbb4054a3ab8317591a0b7320360b444ae" integrity sha512-T1QyPSDCyMXaO3pzBkF96E8xMkiRYbUEZADd29SyPGabqxMViNoii+NcK7eWJAEoU6RZyEm5lVSIjTmcdoB9HA== -"@esbuild/win32-x64@0.20.2": - version "0.20.2" - resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.20.2.tgz#9c907b21e30a52db959ba4f80bb01a0cc403d5cc" - integrity sha512-N49X4lJX27+l9jbLKSqZ6bKNjzQvHaT8IIFUy+YIqmXQdjYCToGWwOItDrfby14c78aDd5NHQl29xingXfCdLQ== +"@esbuild/win32-x64@0.21.4": + version "0.21.4" + resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.21.4.tgz#11ef0398f9abee161193461910a507ef0d4c0c32" + integrity sha512-sywLRD3UK/qRJt0oBwdpYLBibk7KiRfbswmWRDabuncQYSlf8aLEEUor/oP6KRz8KEG+HoiVLBhPRD5JWjS8Sg== "@eslint-community/eslint-utils@^4.2.0", "@eslint-community/eslint-utils@^4.4.0": version "4.4.0" @@ -1020,85 +1020,85 @@ estree-walker "^2.0.2" picomatch "^2.3.1" -"@rollup/rollup-android-arm-eabi@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.17.2.tgz#1a32112822660ee104c5dd3a7c595e26100d4c2d" - integrity sha512-NM0jFxY8bB8QLkoKxIQeObCaDlJKewVlIEkuyYKm5An1tdVZ966w2+MPQ2l8LBZLjR+SgyV+nRkTIunzOYBMLQ== - -"@rollup/rollup-android-arm64@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.17.2.tgz#5aeef206d65ff4db423f3a93f71af91b28662c5b" - integrity sha512-yeX/Usk7daNIVwkq2uGoq2BYJKZY1JfyLTaHO/jaiSwi/lsf8fTFoQW/n6IdAsx5tx+iotu2zCJwz8MxI6D/Bw== - -"@rollup/rollup-darwin-arm64@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.17.2.tgz#6b66aaf003c70454c292cd5f0236ebdc6ffbdf1a" - integrity sha512-kcMLpE6uCwls023+kknm71ug7MZOrtXo+y5p/tsg6jltpDtgQY1Eq5sGfHcQfb+lfuKwhBmEURDga9N0ol4YPw== - -"@rollup/rollup-darwin-x64@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.17.2.tgz#f64fc51ed12b19f883131ccbcea59fc68cbd6c0b" - integrity sha512-AtKwD0VEx0zWkL0ZjixEkp5tbNLzX+FCqGG1SvOu993HnSz4qDI6S4kGzubrEJAljpVkhRSlg5bzpV//E6ysTQ== - -"@rollup/rollup-linux-arm-gnueabihf@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.17.2.tgz#1a7641111be67c10111f7122d1e375d1226cbf14" - integrity sha512-3reX2fUHqN7sffBNqmEyMQVj/CKhIHZd4y631duy0hZqI8Qoqf6lTtmAKvJFYa6bhU95B1D0WgzHkmTg33In0A== - -"@rollup/rollup-linux-arm-musleabihf@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.17.2.tgz#c93fd632923e0fee25aacd2ae414288d0b7455bb" - integrity sha512-uSqpsp91mheRgw96xtyAGP9FW5ChctTFEoXP0r5FAzj/3ZRv3Uxjtc7taRQSaQM/q85KEKjKsZuiZM3GyUivRg== - -"@rollup/rollup-linux-arm64-gnu@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.17.2.tgz#fa531425dd21d058a630947527b4612d9d0b4a4a" - integrity sha512-EMMPHkiCRtE8Wdk3Qhtciq6BndLtstqZIroHiiGzB3C5LDJmIZcSzVtLRbwuXuUft1Cnv+9fxuDtDxz3k3EW2A== - -"@rollup/rollup-linux-arm64-musl@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.17.2.tgz#8acc16f095ceea5854caf7b07e73f7d1802ac5af" - integrity sha512-NMPylUUZ1i0z/xJUIx6VUhISZDRT+uTWpBcjdv0/zkp7b/bQDF+NfnfdzuTiB1G6HTodgoFa93hp0O1xl+/UbA== - -"@rollup/rollup-linux-powerpc64le-gnu@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.17.2.tgz#94e69a8499b5cf368911b83a44bb230782aeb571" - integrity sha512-T19My13y8uYXPw/L/k0JYaX1fJKFT/PWdXiHr8mTbXWxjVF1t+8Xl31DgBBvEKclw+1b00Chg0hxE2O7bTG7GQ== - -"@rollup/rollup-linux-riscv64-gnu@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.17.2.tgz#7ef1c781c7e59e85a6ce261cc95d7f1e0b56db0f" - integrity sha512-BOaNfthf3X3fOWAB+IJ9kxTgPmMqPPH5f5k2DcCsRrBIbWnaJCgX2ll77dV1TdSy9SaXTR5iDXRL8n7AnoP5cg== - -"@rollup/rollup-linux-s390x-gnu@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.17.2.tgz#f15775841c3232fca9b78cd25a7a0512c694b354" - integrity sha512-W0UP/x7bnn3xN2eYMql2T/+wpASLE5SjObXILTMPUBDB/Fg/FxC+gX4nvCfPBCbNhz51C+HcqQp2qQ4u25ok6g== - -"@rollup/rollup-linux-x64-gnu@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.17.2.tgz#b521d271798d037ad70c9f85dd97d25f8a52e811" - integrity sha512-Hy7pLwByUOuyaFC6mAr7m+oMC+V7qyifzs/nW2OJfC8H4hbCzOX07Ov0VFk/zP3kBsELWNFi7rJtgbKYsav9QQ== - -"@rollup/rollup-linux-x64-musl@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.17.2.tgz#9254019cc4baac35800991315d133cc9fd1bf385" - integrity sha512-h1+yTWeYbRdAyJ/jMiVw0l6fOOm/0D1vNLui9iPuqgRGnXA0u21gAqOyB5iHjlM9MMfNOm9RHCQ7zLIzT0x11Q== - -"@rollup/rollup-win32-arm64-msvc@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.17.2.tgz#27f65a89f6f52ee9426ec11e3571038e4671790f" - integrity sha512-tmdtXMfKAjy5+IQsVtDiCfqbynAQE/TQRpWdVataHmhMb9DCoJxp9vLcCBjEQWMiUYxO1QprH/HbY9ragCEFLA== - -"@rollup/rollup-win32-ia32-msvc@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.17.2.tgz#a2fbf8246ed0bb014f078ca34ae6b377a90cb411" - integrity sha512-7II/QCSTAHuE5vdZaQEwJq2ZACkBpQDOmQsE6D6XUbnBHW8IAhm4eTufL6msLJorzrHDFv3CF8oCA/hSIRuZeQ== - -"@rollup/rollup-win32-x64-msvc@4.17.2": - version "4.17.2" - resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.17.2.tgz#5a2d08b81e8064b34242d5cc9973ef8dd1e60503" - integrity sha512-TGGO7v7qOq4CYmSBVEYpI1Y5xDuCEnbVC5Vth8mOsW0gDSzxNrVERPc790IGHsrT2dQSimgMr9Ub3Y1Jci5/8w== +"@rollup/rollup-android-arm-eabi@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.18.0.tgz#bbd0e616b2078cd2d68afc9824d1fadb2f2ffd27" + integrity sha512-Tya6xypR10giZV1XzxmH5wr25VcZSncG0pZIjfePT0OVBvqNEurzValetGNarVrGiq66EBVAFn15iYX4w6FKgQ== + +"@rollup/rollup-android-arm64@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.18.0.tgz#97255ef6384c5f73f4800c0de91f5f6518e21203" + integrity sha512-avCea0RAP03lTsDhEyfy+hpfr85KfyTctMADqHVhLAF3MlIkq83CP8UfAHUssgXTYd+6er6PaAhx/QGv4L1EiA== + +"@rollup/rollup-darwin-arm64@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.18.0.tgz#b6dd74e117510dfe94541646067b0545b42ff096" + integrity sha512-IWfdwU7KDSm07Ty0PuA/W2JYoZ4iTj3TUQjkVsO/6U+4I1jN5lcR71ZEvRh52sDOERdnNhhHU57UITXz5jC1/w== + +"@rollup/rollup-darwin-x64@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.18.0.tgz#e07d76de1cec987673e7f3d48ccb8e106d42c05c" + integrity sha512-n2LMsUz7Ynu7DoQrSQkBf8iNrjOGyPLrdSg802vk6XT3FtsgX6JbE8IHRvposskFm9SNxzkLYGSq9QdpLYpRNA== + +"@rollup/rollup-linux-arm-gnueabihf@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.18.0.tgz#9f1a6d218b560c9d75185af4b8bb42f9f24736b8" + integrity sha512-C/zbRYRXFjWvz9Z4haRxcTdnkPt1BtCkz+7RtBSuNmKzMzp3ZxdM28Mpccn6pt28/UWUCTXa+b0Mx1k3g6NOMA== + +"@rollup/rollup-linux-arm-musleabihf@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.18.0.tgz#53618b92e6ffb642c7b620e6e528446511330549" + integrity sha512-l3m9ewPgjQSXrUMHg93vt0hYCGnrMOcUpTz6FLtbwljo2HluS4zTXFy2571YQbisTnfTKPZ01u/ukJdQTLGh9A== + +"@rollup/rollup-linux-arm64-gnu@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.18.0.tgz#99a7ba5e719d4f053761a698f7b52291cefba577" + integrity sha512-rJ5D47d8WD7J+7STKdCUAgmQk49xuFrRi9pZkWoRD1UeSMakbcepWXPF8ycChBoAqs1pb2wzvbY6Q33WmN2ftw== + +"@rollup/rollup-linux-arm64-musl@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.18.0.tgz#f53db99a45d9bc00ce94db8a35efa7c3c144a58c" + integrity sha512-be6Yx37b24ZwxQ+wOQXXLZqpq4jTckJhtGlWGZs68TgdKXJgw54lUUoFYrg6Zs/kjzAQwEwYbp8JxZVzZLRepQ== + +"@rollup/rollup-linux-powerpc64le-gnu@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.18.0.tgz#cbb0837408fe081ce3435cf3730e090febafc9bf" + integrity sha512-hNVMQK+qrA9Todu9+wqrXOHxFiD5YmdEi3paj6vP02Kx1hjd2LLYR2eaN7DsEshg09+9uzWi2W18MJDlG0cxJA== + +"@rollup/rollup-linux-riscv64-gnu@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.18.0.tgz#8ed09c1d1262ada4c38d791a28ae0fea28b80cc9" + integrity sha512-ROCM7i+m1NfdrsmvwSzoxp9HFtmKGHEqu5NNDiZWQtXLA8S5HBCkVvKAxJ8U+CVctHwV2Gb5VUaK7UAkzhDjlg== + +"@rollup/rollup-linux-s390x-gnu@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.18.0.tgz#938138d3c8e0c96f022252a28441dcfb17afd7ec" + integrity sha512-0UyyRHyDN42QL+NbqevXIIUnKA47A+45WyasO+y2bGJ1mhQrfrtXUpTxCOrfxCR4esV3/RLYyucGVPiUsO8xjg== + +"@rollup/rollup-linux-x64-gnu@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.18.0.tgz#1a7481137a54740bee1ded4ae5752450f155d942" + integrity sha512-xuglR2rBVHA5UsI8h8UbX4VJ470PtGCf5Vpswh7p2ukaqBGFTnsfzxUBetoWBWymHMxbIG0Cmx7Y9qDZzr648w== + +"@rollup/rollup-linux-x64-musl@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.18.0.tgz#f1186afc601ac4f4fc25fac4ca15ecbee3a1874d" + integrity sha512-LKaqQL9osY/ir2geuLVvRRs+utWUNilzdE90TpyoX0eNqPzWjRm14oMEE+YLve4k/NAqCdPkGYDaDF5Sw+xBfg== + +"@rollup/rollup-win32-arm64-msvc@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.18.0.tgz#ed6603e93636a96203c6915be4117245c1bd2daf" + integrity sha512-7J6TkZQFGo9qBKH0pk2cEVSRhJbL6MtfWxth7Y5YmZs57Pi+4x6c2dStAUvaQkHQLnEQv1jzBUW43GvZW8OFqA== + +"@rollup/rollup-win32-ia32-msvc@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.18.0.tgz#14e0b404b1c25ebe6157a15edb9c46959ba74c54" + integrity sha512-Txjh+IxBPbkUB9+SXZMpv+b/vnTEtFyfWZgJ6iyCmt2tdx0OF5WhFowLmnh8ENGNpfUlUZkdI//4IEmhwPieNg== + +"@rollup/rollup-win32-x64-msvc@4.18.0": + version "4.18.0" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.18.0.tgz#5d694d345ce36b6ecf657349e03eb87297e68da4" + integrity sha512-UOo5FdvOL0+eIVTgS4tIdbW+TtnBLWg1YBCcU2KWM7nuNwRz9bksDX1bekJJCpu25N1DVWaCwnT39dVQxzqS8g== "@rollup/stream@3.0.1": version "3.0.1" @@ -1124,91 +1124,91 @@ dependencies: "@sinonjs/commons" "^3.0.0" -"@swc/core-darwin-arm64@1.4.17": - version "1.4.17" - resolved "https://registry.yarnpkg.com/@swc/core-darwin-arm64/-/core-darwin-arm64-1.4.17.tgz#e62fa7f247bdd1c0c50a3f99722da4dd098c7c67" - integrity sha512-HVl+W4LezoqHBAYg2JCqR+s9ife9yPfgWSj37iIawLWzOmuuJ7jVdIB7Ee2B75bEisSEKyxRlTl6Y1Oq3owBgw== - -"@swc/core-darwin-x64@1.4.17": - version "1.4.17" - resolved "https://registry.yarnpkg.com/@swc/core-darwin-x64/-/core-darwin-x64-1.4.17.tgz#1145cbb7575e317204ed3a7d0274bd26fe9ffab6" - integrity sha512-WYRO9Fdzq4S/he8zjW5I95G1zcvyd9yyD3Tgi4/ic84P5XDlSMpBDpBLbr/dCPjmSg7aUXxNQqKqGkl6dQxYlA== - -"@swc/core-linux-arm-gnueabihf@1.4.17": - version "1.4.17" - resolved "https://registry.yarnpkg.com/@swc/core-linux-arm-gnueabihf/-/core-linux-arm-gnueabihf-1.4.17.tgz#7145b3ada5cf9b748eaacbc9a7c7037ba0fb26bb" - integrity sha512-cgbvpWOvtMH0XFjvwppUCR+Y+nf6QPaGu6AQ5hqCP+5Lv2zO5PG0RfasC4zBIjF53xgwEaaWmGP5/361P30X8Q== - -"@swc/core-linux-arm64-gnu@1.4.17": - version "1.4.17" - resolved "https://registry.yarnpkg.com/@swc/core-linux-arm64-gnu/-/core-linux-arm64-gnu-1.4.17.tgz#5c0833ef132af17bd3cbdf2253f35b57c0cf62bb" - integrity sha512-l7zHgaIY24cF9dyQ/FOWbmZDsEj2a9gRFbmgx2u19e3FzOPuOnaopFj0fRYXXKCmtdx+anD750iBIYnTR+pq/Q== - -"@swc/core-linux-arm64-musl@1.4.17": - version "1.4.17" - resolved "https://registry.yarnpkg.com/@swc/core-linux-arm64-musl/-/core-linux-arm64-musl-1.4.17.tgz#5bfe81eb23c905f04b669a7d2b060a147a263483" - integrity sha512-qhH4gr9gAlVk8MBtzXbzTP3BJyqbAfUOATGkyUtohh85fPXQYuzVlbExix3FZXTwFHNidGHY8C+ocscI7uDaYw== - -"@swc/core-linux-x64-gnu@1.4.17": - version "1.4.17" - resolved "https://registry.yarnpkg.com/@swc/core-linux-x64-gnu/-/core-linux-x64-gnu-1.4.17.tgz#a0c19bc9635e86ebd1c7f8e9e026503d1a1bf83d" - integrity sha512-vRDFATL1oN5oZMImkwbgSHEkp8xG1ofEASBypze01W1Tqto8t+yo6gsp69wzCZBlxldsvPpvFZW55Jq0Rn+UnA== - -"@swc/core-linux-x64-musl@1.4.17": - version "1.4.17" - resolved "https://registry.yarnpkg.com/@swc/core-linux-x64-musl/-/core-linux-x64-musl-1.4.17.tgz#2179b9536235a3b02a46997ddb1c178dfadf1667" - integrity sha512-zQNPXAXn3nmPqv54JVEN8k2JMEcMTQ6veVuU0p5O+A7KscJq+AGle/7ZQXzpXSfUCXlLMX4wvd+rwfGhh3J4cw== - -"@swc/core-win32-arm64-msvc@1.4.17": - version "1.4.17" - resolved "https://registry.yarnpkg.com/@swc/core-win32-arm64-msvc/-/core-win32-arm64-msvc-1.4.17.tgz#3004a431c836c6b16b4660ea2425dde467a8ee36" - integrity sha512-z86n7EhOwyzxwm+DLE5NoLkxCTme2lq7QZlDjbQyfCxOt6isWz8rkW5QowTX8w9Rdmk34ncrjSLvnHOeLY17+w== - -"@swc/core-win32-ia32-msvc@1.4.17": - version "1.4.17" - resolved "https://registry.yarnpkg.com/@swc/core-win32-ia32-msvc/-/core-win32-ia32-msvc-1.4.17.tgz#59155485d5307fb2a267e5acb215e0f440b6f48f" - integrity sha512-JBwuSTJIgiJJX6wtr4wmXbfvOswHFj223AumUrK544QV69k60FJ9q2adPW9Csk+a8wm1hLxq4HKa2K334UHJ/g== - -"@swc/core-win32-x64-msvc@1.4.17": - version "1.4.17" - resolved "https://registry.yarnpkg.com/@swc/core-win32-x64-msvc/-/core-win32-x64-msvc-1.4.17.tgz#b98f25fc277fb0e319f25f9fd00a82023662716b" - integrity sha512-jFkOnGQamtVDBm3MF5Kq1lgW8vx4Rm1UvJWRUfg+0gx7Uc3Jp3QMFeMNw/rDNQYRDYPG3yunCC+2463ycd5+dg== - -"@swc/core@1.4.17": - version "1.4.17" - resolved "https://registry.yarnpkg.com/@swc/core/-/core-1.4.17.tgz#3ea4180fa5c54282b284006a6de1263ef1cf887f" - integrity sha512-tq+mdWvodMBNBBZbwFIMTVGYHe9N7zvEaycVVjfvAx20k1XozHbHhRv+9pEVFJjwRxLdXmtvFZd3QZHRAOpoNQ== - dependencies: - "@swc/counter" "^0.1.2" - "@swc/types" "^0.1.5" +"@swc/core-darwin-arm64@1.5.24": + version "1.5.24" + resolved "https://registry.yarnpkg.com/@swc/core-darwin-arm64/-/core-darwin-arm64-1.5.24.tgz#71875695bc617e57c2d93352f48317b4c41e0240" + integrity sha512-M7oLOcC0sw+UTyAuL/9uyB9GeO4ZpaBbH76JSH6g1m0/yg7LYJZGRmplhDmwVSDAR5Fq4Sjoi1CksmmGkgihGA== + +"@swc/core-darwin-x64@1.5.24": + version "1.5.24" + resolved "https://registry.yarnpkg.com/@swc/core-darwin-x64/-/core-darwin-x64-1.5.24.tgz#6b4c3eb9b21ab50b7324a82c9497ffeb2e8e0a57" + integrity sha512-MfcFjGGYognpSBSos2pYUNYJSmqEhuw5ceGr6qAdME7ddbjGXliza4W6FggsM+JnWwpqa31+e7/R+GetW4WkaQ== + +"@swc/core-linux-arm-gnueabihf@1.5.24": + version "1.5.24" + resolved "https://registry.yarnpkg.com/@swc/core-linux-arm-gnueabihf/-/core-linux-arm-gnueabihf-1.5.24.tgz#5730ed6ad86afe4ee8df04ee6f21430daead186c" + integrity sha512-amI2pwtcWV3E/m/nf+AQtn1LWDzKLZyjCmWd3ms7QjEueWYrY8cU1Y4Wp7wNNsxIoPOi8zek1Uj2wwFD/pttNQ== + +"@swc/core-linux-arm64-gnu@1.5.24": + version "1.5.24" + resolved "https://registry.yarnpkg.com/@swc/core-linux-arm64-gnu/-/core-linux-arm64-gnu-1.5.24.tgz#0a2478e8601391aa88f82bfece1dbc60d27cbcfd" + integrity sha512-sTSvmqMmgT1ynH/nP75Pc51s+iT4crZagHBiDOf5cq+kudUYjda9lWMs7xkXB/TUKFHPCRK0HGunl8bkwiIbuw== + +"@swc/core-linux-arm64-musl@1.5.24": + version "1.5.24" + resolved "https://registry.yarnpkg.com/@swc/core-linux-arm64-musl/-/core-linux-arm64-musl-1.5.24.tgz#e0199092dc611ca75f8a92dcea17de44e38f3fbf" + integrity sha512-vd2/hfOBGbrX21FxsFdXCUaffjkHvlZkeE2UMRajdXifwv79jqOHIJg3jXG1F3ZrhCghCzirFts4tAZgcG8XWg== + +"@swc/core-linux-x64-gnu@1.5.24": + version "1.5.24" + resolved "https://registry.yarnpkg.com/@swc/core-linux-x64-gnu/-/core-linux-x64-gnu-1.5.24.tgz#1fe347c9f28457c593f2fda5b0d4904a2b105ecd" + integrity sha512-Zrdzi7NqzQxm2BvAG5KyOSBEggQ7ayrxh599AqqevJmsUXJ8o2nMiWQOBvgCGp7ye+Biz3pvZn1EnRzAp+TpUg== + +"@swc/core-linux-x64-musl@1.5.24": + version "1.5.24" + resolved "https://registry.yarnpkg.com/@swc/core-linux-x64-musl/-/core-linux-x64-musl-1.5.24.tgz#bf6ac583fac211d704d2d78cfd0b7bf751268f5e" + integrity sha512-1F8z9NRi52jdZQCGc5sflwYSctL6omxiVmIFVp8TC9nngjQKc00TtX/JC2Eo2HwvgupkFVl5YQJidAck9YtmJw== + +"@swc/core-win32-arm64-msvc@1.5.24": + version "1.5.24" + resolved "https://registry.yarnpkg.com/@swc/core-win32-arm64-msvc/-/core-win32-arm64-msvc-1.5.24.tgz#41b9faf4db69cc08a43c3a176df2a7b94d765637" + integrity sha512-cKpP7KvS6Xr0jFSTBXY53HZX/YfomK5EMQYpCVDOvfsZeYHN20sQSKXfpVLvA/q2igVt1zzy1XJcOhpJcgiKLg== + +"@swc/core-win32-ia32-msvc@1.5.24": + version "1.5.24" + resolved "https://registry.yarnpkg.com/@swc/core-win32-ia32-msvc/-/core-win32-ia32-msvc-1.5.24.tgz#e123ad00e3b28d567d3851a86697fb3c54ed817a" + integrity sha512-IoPWfi0iwqjZuf7gE223+B97/ZwkKbu7qL5KzGP7g3hJrGSKAvv7eC5Y9r2iKKtLKyv5R/T6Ho0kFR/usi7rHw== + +"@swc/core-win32-x64-msvc@1.5.24": + version "1.5.24" + resolved "https://registry.yarnpkg.com/@swc/core-win32-x64-msvc/-/core-win32-x64-msvc-1.5.24.tgz#21fb87b1981253039e6d45255e31a875f446e397" + integrity sha512-zHgF2k1uVJL8KIW+PnVz1To4a3Cz9THbh2z2lbehaF/gKHugH4c3djBozU4das1v35KOqf5jWIEviBLql2wDLQ== + +"@swc/core@1.5.24": + version "1.5.24" + resolved "https://registry.yarnpkg.com/@swc/core/-/core-1.5.24.tgz#9ecb4601cb6a4fb19f227ec5fb59d07e23347dca" + integrity sha512-Eph9zvO4xvqWZGVzTdtdEJ0Vqf0VIML/o/e4Qd2RLOqtfgnlRi7avmMu5C0oqciJ0tk+hqdUKVUZ4JPoPaiGvQ== + dependencies: + "@swc/counter" "^0.1.3" + "@swc/types" "^0.1.7" optionalDependencies: - "@swc/core-darwin-arm64" "1.4.17" - "@swc/core-darwin-x64" "1.4.17" - "@swc/core-linux-arm-gnueabihf" "1.4.17" - "@swc/core-linux-arm64-gnu" "1.4.17" - "@swc/core-linux-arm64-musl" "1.4.17" - "@swc/core-linux-x64-gnu" "1.4.17" - "@swc/core-linux-x64-musl" "1.4.17" - "@swc/core-win32-arm64-msvc" "1.4.17" - "@swc/core-win32-ia32-msvc" "1.4.17" - "@swc/core-win32-x64-msvc" "1.4.17" - -"@swc/counter@^0.1.2", "@swc/counter@^0.1.3": + "@swc/core-darwin-arm64" "1.5.24" + "@swc/core-darwin-x64" "1.5.24" + "@swc/core-linux-arm-gnueabihf" "1.5.24" + "@swc/core-linux-arm64-gnu" "1.5.24" + "@swc/core-linux-arm64-musl" "1.5.24" + "@swc/core-linux-x64-gnu" "1.5.24" + "@swc/core-linux-x64-musl" "1.5.24" + "@swc/core-win32-arm64-msvc" "1.5.24" + "@swc/core-win32-ia32-msvc" "1.5.24" + "@swc/core-win32-x64-msvc" "1.5.24" + +"@swc/counter@^0.1.3": version "0.1.3" resolved "https://registry.yarnpkg.com/@swc/counter/-/counter-0.1.3.tgz#cc7463bd02949611c6329596fccd2b0ec782b0e9" integrity sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ== -"@swc/helpers@^0.5.10": +"@swc/helpers@^0.5.11": version "0.5.11" resolved "https://registry.yarnpkg.com/@swc/helpers/-/helpers-0.5.11.tgz#5bab8c660a6e23c13b2d23fcd1ee44a2db1b0cb7" integrity sha512-YNlnKRWF2sVojTpIyzwou9XoTNbzbzONwRhOoniEioF1AtaitTvVZblaQRrAzChWQ1bLYyYSWzM18y4WwgzJ+A== dependencies: tslib "^2.4.0" -"@swc/types@^0.1.5": - version "0.1.6" - resolved "https://registry.yarnpkg.com/@swc/types/-/types-0.1.6.tgz#2f13f748995b247d146de2784d3eb7195410faba" - integrity sha512-/JLo/l2JsT/LRd80C3HfbmVpxOAJ11FO2RCEslFrgzLltoP9j8XIbsyDcfCt2WWyX+CM96rBoNM+IToAkFOugg== +"@swc/types@^0.1.7": + version "0.1.7" + resolved "https://registry.yarnpkg.com/@swc/types/-/types-0.1.7.tgz#ea5d658cf460abff51507ca8d26e2d391bafb15e" + integrity sha512-scHWahbHF0eyj3JsxG9CFJgFdFNaVQCNAimBlT6PzS3n/HptxqREjsm4OH6AN3lYcffZYSPxXW8ua2BEHp0lJQ== dependencies: "@swc/counter" "^0.1.3" @@ -1348,7 +1348,7 @@ expect "^29.0.0" pretty-format "^29.0.0" -"@types/json-schema@*", "@types/json-schema@^7.0.12", "@types/json-schema@^7.0.15", "@types/json-schema@^7.0.8": +"@types/json-schema@*", "@types/json-schema@^7.0.8": version "7.0.15" resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.15.tgz#596a1747233694d50f6ad8a7869fcb6f56cf5841" integrity sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA== @@ -1370,17 +1370,19 @@ dependencies: "@types/node" "*" -"@types/node@*", "@types/node@^20.12.7": +"@types/node@*": version "20.12.8" resolved "https://registry.yarnpkg.com/@types/node/-/node-20.12.8.tgz#35897bf2bfe3469847ab04634636de09552e8256" integrity sha512-NU0rJLJnshZWdE/097cdCBbyW1h4hEg0xpovcoAQYHl8dnEyp/NAOiE45pvc+Bd1Dt+2r94v2eGFpQJ4R7g+2w== dependencies: undici-types "~5.26.4" -"@types/node@^13.7.4": - version "13.13.52" - resolved "https://registry.yarnpkg.com/@types/node/-/node-13.13.52.tgz#03c13be70b9031baaed79481c0c0cfb0045e53f7" - integrity sha512-s3nugnZumCC//n4moGGe6tkNMyYEdaDBitVjwPxXmR5lnMG5dHePinH2EdxkG3Rh1ghFHHixAG4NJhpJW1rthQ== +"@types/node@>=13.7.4", "@types/node@^20.13.0": + version "20.13.0" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.13.0.tgz#011a76bc1e71ae9a026dddcfd7039084f752c4b6" + integrity sha512-FM6AOb3khNkNIXPnHFDYaHerSv8uN22C91z098AnGccVu+Pcdhi+pNUFDi0iLmPIsVE0JBD0KVS7mzUYt4nRzQ== + dependencies: + undici-types "~5.26.4" "@types/normalize-package-data@^2.4.0": version "2.4.4" @@ -1392,11 +1394,6 @@ resolved "https://registry.yarnpkg.com/@types/resolve/-/resolve-1.20.2.tgz#97d26e00cd4a0423b4af620abecf3e6f442b7975" integrity sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q== -"@types/semver@^7.5.0", "@types/semver@^7.5.8": - version "7.5.8" - resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.8.tgz#8268a8c57a3e4abd25c165ecd36237db7948a55e" - integrity sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ== - "@types/stack-utils@^2.0.0": version "2.0.3" resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-2.0.3.tgz#6209321eb2c1712a7e7466422b8cb1fc0d9dd5d8" @@ -1429,91 +1426,62 @@ dependencies: "@types/yargs-parser" "*" -"@typescript-eslint/eslint-plugin@7.8.0": - version "7.8.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.8.0.tgz#c78e309fe967cb4de05b85cdc876fb95f8e01b6f" - integrity sha512-gFTT+ezJmkwutUPmB0skOj3GZJtlEGnlssems4AjkVweUPGj7jRwwqg0Hhg7++kPGJqKtTYx+R05Ftww372aIg== +"@typescript-eslint/eslint-plugin@7.11.0": + version "7.11.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.11.0.tgz#f90f0914657ead08e1c75f66939c926edeab42dd" + integrity sha512-P+qEahbgeHW4JQ/87FuItjBj8O3MYv5gELDzr8QaQ7fsll1gSMTYb6j87MYyxwf3DtD7uGFB9ShwgmCJB5KmaQ== dependencies: "@eslint-community/regexpp" "^4.10.0" - "@typescript-eslint/scope-manager" "7.8.0" - "@typescript-eslint/type-utils" "7.8.0" - "@typescript-eslint/utils" "7.8.0" - "@typescript-eslint/visitor-keys" "7.8.0" - debug "^4.3.4" + "@typescript-eslint/scope-manager" "7.11.0" + "@typescript-eslint/type-utils" "7.11.0" + "@typescript-eslint/utils" "7.11.0" + "@typescript-eslint/visitor-keys" "7.11.0" graphemer "^1.4.0" ignore "^5.3.1" natural-compare "^1.4.0" - semver "^7.6.0" ts-api-utils "^1.3.0" -"@typescript-eslint/parser@7.8.0": - version "7.8.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-7.8.0.tgz#1e1db30c8ab832caffee5f37e677dbcb9357ddc8" - integrity sha512-KgKQly1pv0l4ltcftP59uQZCi4HUYswCLbTqVZEJu7uLX8CTLyswqMLqLN+2QFz4jCptqWVV4SB7vdxcH2+0kQ== +"@typescript-eslint/parser@7.11.0": + version "7.11.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-7.11.0.tgz#525ad8bee54a8f015f134edd241d91b84ab64839" + integrity sha512-yimw99teuaXVWsBcPO1Ais02kwJ1jmNA1KxE7ng0aT7ndr1pT1wqj0OJnsYVGKKlc4QJai86l/025L6z8CljOg== dependencies: - "@typescript-eslint/scope-manager" "7.8.0" - "@typescript-eslint/types" "7.8.0" - "@typescript-eslint/typescript-estree" "7.8.0" - "@typescript-eslint/visitor-keys" "7.8.0" + "@typescript-eslint/scope-manager" "7.11.0" + "@typescript-eslint/types" "7.11.0" + "@typescript-eslint/typescript-estree" "7.11.0" + "@typescript-eslint/visitor-keys" "7.11.0" debug "^4.3.4" -"@typescript-eslint/scope-manager@6.21.0": - version "6.21.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-6.21.0.tgz#ea8a9bfc8f1504a6ac5d59a6df308d3a0630a2b1" - integrity sha512-OwLUIWZJry80O99zvqXVEioyniJMa+d2GrqpUTqi5/v5D5rOrppJVBPa0yKCblcigC0/aYAzxxqQ1B+DS2RYsg== - dependencies: - "@typescript-eslint/types" "6.21.0" - "@typescript-eslint/visitor-keys" "6.21.0" - -"@typescript-eslint/scope-manager@7.8.0": - version "7.8.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-7.8.0.tgz#bb19096d11ec6b87fb6640d921df19b813e02047" - integrity sha512-viEmZ1LmwsGcnr85gIq+FCYI7nO90DVbE37/ll51hjv9aG+YZMb4WDE2fyWpUR4O/UrhGRpYXK/XajcGTk2B8g== +"@typescript-eslint/scope-manager@7.11.0": + version "7.11.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-7.11.0.tgz#cf5619b01de62a226a59add15a02bde457335d1d" + integrity sha512-27tGdVEiutD4POirLZX4YzT180vevUURJl4wJGmm6TrQoiYwuxTIY98PBp6L2oN+JQxzE0URvYlzJaBHIekXAw== dependencies: - "@typescript-eslint/types" "7.8.0" - "@typescript-eslint/visitor-keys" "7.8.0" + "@typescript-eslint/types" "7.11.0" + "@typescript-eslint/visitor-keys" "7.11.0" -"@typescript-eslint/type-utils@7.8.0": - version "7.8.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-7.8.0.tgz#9de166f182a6e4d1c5da76e94880e91831e3e26f" - integrity sha512-H70R3AefQDQpz9mGv13Uhi121FNMh+WEaRqcXTX09YEDky21km4dV1ZXJIp8QjXc4ZaVkXVdohvWDzbnbHDS+A== +"@typescript-eslint/type-utils@7.11.0": + version "7.11.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-7.11.0.tgz#ac216697d649084fedf4a910347b9642bd0ff099" + integrity sha512-WmppUEgYy+y1NTseNMJ6mCFxt03/7jTOy08bcg7bxJJdsM4nuhnchyBbE8vryveaJUf62noH7LodPSo5Z0WUCg== dependencies: - "@typescript-eslint/typescript-estree" "7.8.0" - "@typescript-eslint/utils" "7.8.0" + "@typescript-eslint/typescript-estree" "7.11.0" + "@typescript-eslint/utils" "7.11.0" debug "^4.3.4" ts-api-utils "^1.3.0" -"@typescript-eslint/types@6.21.0": - version "6.21.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-6.21.0.tgz#205724c5123a8fef7ecd195075fa6e85bac3436d" - integrity sha512-1kFmZ1rOm5epu9NZEZm1kckCDGj5UJEf7P1kliH4LKu/RkwpsfqqGmY2OOcUs18lSlQBKLDYBOGxRVtrMN5lpg== - -"@typescript-eslint/types@7.8.0": - version "7.8.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-7.8.0.tgz#1fd2577b3ad883b769546e2d1ef379f929a7091d" - integrity sha512-wf0peJ+ZGlcH+2ZS23aJbOv+ztjeeP8uQ9GgwMJGVLx/Nj9CJt17GWgWWoSmoRVKAX2X+7fzEnAjxdvK2gqCLw== +"@typescript-eslint/types@7.11.0": + version "7.11.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-7.11.0.tgz#5e9702a5e8b424b7fc690e338d359939257d6722" + integrity sha512-MPEsDRZTyCiXkD4vd3zywDCifi7tatc4K37KqTprCvaXptP7Xlpdw0NR2hRJTetG5TxbWDB79Ys4kLmHliEo/w== -"@typescript-eslint/typescript-estree@6.21.0": - version "6.21.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-6.21.0.tgz#c47ae7901db3b8bddc3ecd73daff2d0895688c46" - integrity sha512-6npJTkZcO+y2/kr+z0hc4HwNfrrP4kNYh57ek7yCNlrBjWQ1Y0OS7jiZTkgumrvkX5HkEKXFZkkdFNkaW2wmUQ== +"@typescript-eslint/typescript-estree@7.11.0": + version "7.11.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-7.11.0.tgz#7cbc569bc7336c3a494ceaf8204fdee5d5dbb7fa" + integrity sha512-cxkhZ2C/iyi3/6U9EPc5y+a6csqHItndvN/CzbNXTNrsC3/ASoYQZEt9uMaEp+xFNjasqQyszp5TumAVKKvJeQ== dependencies: - "@typescript-eslint/types" "6.21.0" - "@typescript-eslint/visitor-keys" "6.21.0" - debug "^4.3.4" - globby "^11.1.0" - is-glob "^4.0.3" - minimatch "9.0.3" - semver "^7.5.4" - ts-api-utils "^1.0.1" - -"@typescript-eslint/typescript-estree@7.8.0": - version "7.8.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-7.8.0.tgz#b028a9226860b66e623c1ee55cc2464b95d2987c" - integrity sha512-5pfUCOwK5yjPaJQNy44prjCwtr981dO8Qo9J9PwYXZ0MosgAbfEMB008dJ5sNo3+/BN6ytBPuSvXUg9SAqB0dg== - dependencies: - "@typescript-eslint/types" "7.8.0" - "@typescript-eslint/visitor-keys" "7.8.0" + "@typescript-eslint/types" "7.11.0" + "@typescript-eslint/visitor-keys" "7.11.0" debug "^4.3.4" globby "^11.1.0" is-glob "^4.0.3" @@ -1521,46 +1489,22 @@ semver "^7.6.0" ts-api-utils "^1.3.0" -"@typescript-eslint/utils@7.8.0": - version "7.8.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-7.8.0.tgz#57a79f9c0c0740ead2f622e444cfaeeb9fd047cd" - integrity sha512-L0yFqOCflVqXxiZyXrDr80lnahQfSOfc9ELAAZ75sqicqp2i36kEZZGuUymHNFoYOqxRT05up760b4iGsl02nQ== +"@typescript-eslint/utils@7.11.0", "@typescript-eslint/utils@^6.0.0 || ^7.0.0": + version "7.11.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-7.11.0.tgz#524f047f2209959424c3ef689b0d83b3bc09919c" + integrity sha512-xlAWwPleNRHwF37AhrZurOxA1wyXowW4PqVXZVUNCLjB48CqdPJoJWkrpH2nij9Q3Lb7rtWindtoXwxjxlKKCA== dependencies: "@eslint-community/eslint-utils" "^4.4.0" - "@types/json-schema" "^7.0.15" - "@types/semver" "^7.5.8" - "@typescript-eslint/scope-manager" "7.8.0" - "@typescript-eslint/types" "7.8.0" - "@typescript-eslint/typescript-estree" "7.8.0" - semver "^7.6.0" - -"@typescript-eslint/utils@^6.0.0": - version "6.21.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-6.21.0.tgz#4714e7a6b39e773c1c8e97ec587f520840cd8134" - integrity sha512-NfWVaC8HP9T8cbKQxHcsJBY5YE1O33+jpMwN45qzWWaPDZgLIbo12toGMWnmhvCpd3sIxkpDw3Wv1B3dYrbDQQ== - dependencies: - "@eslint-community/eslint-utils" "^4.4.0" - "@types/json-schema" "^7.0.12" - "@types/semver" "^7.5.0" - "@typescript-eslint/scope-manager" "6.21.0" - "@typescript-eslint/types" "6.21.0" - "@typescript-eslint/typescript-estree" "6.21.0" - semver "^7.5.4" + "@typescript-eslint/scope-manager" "7.11.0" + "@typescript-eslint/types" "7.11.0" + "@typescript-eslint/typescript-estree" "7.11.0" -"@typescript-eslint/visitor-keys@6.21.0": - version "6.21.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-6.21.0.tgz#87a99d077aa507e20e238b11d56cc26ade45fe47" - integrity sha512-JJtkDduxLi9bivAB+cYOVMtbkqdPOhZ+ZI5LC47MIRrDV4Yn2o+ZnW10Nkmr28xRpSpdJ6Sm42Hjf2+REYXm0A== +"@typescript-eslint/visitor-keys@7.11.0": + version "7.11.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-7.11.0.tgz#2c50cd292e67645eec05ac0830757071b4a4d597" + integrity sha512-7syYk4MzjxTEk0g/w3iqtgxnFQspDJfn6QKD36xMuuhTzjcxY7F8EmBLnALjVyaOF1/bVocu3bS/2/F7rXrveQ== dependencies: - "@typescript-eslint/types" "6.21.0" - eslint-visitor-keys "^3.4.1" - -"@typescript-eslint/visitor-keys@7.8.0": - version "7.8.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-7.8.0.tgz#7285aab991da8bee411a42edbd5db760d22fdd91" - integrity sha512-q4/gibTNBQNA0lGyYQCmWRS5D15n8rXh4QjK3KV+MBPlTYHpfBUT3D3PaPR/HeNiI9W6R7FvlkcGhNyAoP+caA== - dependencies: - "@typescript-eslint/types" "7.8.0" + "@typescript-eslint/types" "7.11.0" eslint-visitor-keys "^3.4.3" "@ungap/structured-clone@^1.2.0": @@ -2960,34 +2904,34 @@ esbuild-plugin-alias@0.2.1: resolved "https://registry.yarnpkg.com/esbuild-plugin-alias/-/esbuild-plugin-alias-0.2.1.tgz#45a86cb941e20e7c2bc68a2bea53562172494fcb" integrity sha512-jyfL/pwPqaFXyKnj8lP8iLk6Z0m099uXR45aSN8Av1XD4vhvQutxxPzgA2bTcAwQpa1zCXDcWOlhFgyP3GKqhQ== -esbuild@0.20.2: - version "0.20.2" - resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.20.2.tgz#9d6b2386561766ee6b5a55196c6d766d28c87ea1" - integrity sha512-WdOOppmUNU+IbZ0PaDiTst80zjnrOkyJNHoKupIcVyU8Lvla3Ugx94VzkQ32Ijqd7UhHJy75gNWDMUekcrSJ6g== +esbuild@0.21.4: + version "0.21.4" + resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.21.4.tgz#ceb501def8edb12a5bfd9c55f3a96db698edf022" + integrity sha512-sFMcNNrj+Q0ZDolrp5pDhH0nRPN9hLIM3fRPwgbLYJeSHHgnXSnbV3xYgSVuOeLWH9c73VwmEverVzupIv5xuA== optionalDependencies: - "@esbuild/aix-ppc64" "0.20.2" - "@esbuild/android-arm" "0.20.2" - "@esbuild/android-arm64" "0.20.2" - "@esbuild/android-x64" "0.20.2" - "@esbuild/darwin-arm64" "0.20.2" - "@esbuild/darwin-x64" "0.20.2" - "@esbuild/freebsd-arm64" "0.20.2" - "@esbuild/freebsd-x64" "0.20.2" - "@esbuild/linux-arm" "0.20.2" - "@esbuild/linux-arm64" "0.20.2" - "@esbuild/linux-ia32" "0.20.2" - "@esbuild/linux-loong64" "0.20.2" - "@esbuild/linux-mips64el" "0.20.2" - "@esbuild/linux-ppc64" "0.20.2" - "@esbuild/linux-riscv64" "0.20.2" - "@esbuild/linux-s390x" "0.20.2" - "@esbuild/linux-x64" "0.20.2" - "@esbuild/netbsd-x64" "0.20.2" - "@esbuild/openbsd-x64" "0.20.2" - "@esbuild/sunos-x64" "0.20.2" - "@esbuild/win32-arm64" "0.20.2" - "@esbuild/win32-ia32" "0.20.2" - "@esbuild/win32-x64" "0.20.2" + "@esbuild/aix-ppc64" "0.21.4" + "@esbuild/android-arm" "0.21.4" + "@esbuild/android-arm64" "0.21.4" + "@esbuild/android-x64" "0.21.4" + "@esbuild/darwin-arm64" "0.21.4" + "@esbuild/darwin-x64" "0.21.4" + "@esbuild/freebsd-arm64" "0.21.4" + "@esbuild/freebsd-x64" "0.21.4" + "@esbuild/linux-arm" "0.21.4" + "@esbuild/linux-arm64" "0.21.4" + "@esbuild/linux-ia32" "0.21.4" + "@esbuild/linux-loong64" "0.21.4" + "@esbuild/linux-mips64el" "0.21.4" + "@esbuild/linux-ppc64" "0.21.4" + "@esbuild/linux-riscv64" "0.21.4" + "@esbuild/linux-s390x" "0.21.4" + "@esbuild/linux-x64" "0.21.4" + "@esbuild/netbsd-x64" "0.21.4" + "@esbuild/openbsd-x64" "0.21.4" + "@esbuild/sunos-x64" "0.21.4" + "@esbuild/win32-arm64" "0.21.4" + "@esbuild/win32-ia32" "0.21.4" + "@esbuild/win32-x64" "0.21.4" esbuild@^0.19.6: version "0.19.12" @@ -3043,12 +2987,12 @@ escape-string-regexp@^4.0.0: resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz#14ba83a5d373e3d311e5afca29cf5bfad965bf34" integrity sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA== -eslint-plugin-jest@28.4.0: - version "28.4.0" - resolved "https://registry.yarnpkg.com/eslint-plugin-jest/-/eslint-plugin-jest-28.4.0.tgz#213be88f799a35ca9d63ce1a30081bb32b8da765" - integrity sha512-ORVHiFPC8RQxHLyQJ37MxNilK9k+cPzjHz65T8gAbpYZunGutXvKqwfM3WXBCvFDF1QBeYJJu9LB/i5cuXBs+g== +eslint-plugin-jest@28.5.0: + version "28.5.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-jest/-/eslint-plugin-jest-28.5.0.tgz#b497b795de37f671eaccd38bd83030186ff5dc8d" + integrity sha512-6np6DGdmNq/eBbA7HOUNV8fkfL86PYwBfwyb8n23FXgJNTR8+ot3smRHjza9LGsBBZRypK3qyF79vMjohIL8eQ== dependencies: - "@typescript-eslint/utils" "^6.0.0" + "@typescript-eslint/utils" "^6.0.0 || ^7.0.0" eslint-plugin-unicorn@52.0.0: version "52.0.0" @@ -3663,16 +3607,16 @@ glob-watcher@^5.0.3: normalize-path "^3.0.0" object.defaults "^1.1.0" -glob@10.3.12: - version "10.3.12" - resolved "https://registry.yarnpkg.com/glob/-/glob-10.3.12.tgz#3a65c363c2e9998d220338e88a5f6ac97302960b" - integrity sha512-TCNv8vJ+xz4QiqTpfOJA7HvYv+tNIRHKfUWw/q+v2jdgN4ebz+KY9tGx5J4rHP0o84mNP+ApH66HRX8us3Khqg== +glob@10.4.1: + version "10.4.1" + resolved "https://registry.yarnpkg.com/glob/-/glob-10.4.1.tgz#0cfb01ab6a6b438177bfe6a58e2576f6efe909c2" + integrity sha512-2jelhlq3E4ho74ZyVLN03oKdAZVUa6UDZzFLVH1H7dnoax+y9qyaq8zBkfDIggjniU19z0wU18y16jMB2eyVIw== dependencies: foreground-child "^3.1.0" - jackspeak "^2.3.6" - minimatch "^9.0.1" - minipass "^7.0.4" - path-scurry "^1.10.2" + jackspeak "^3.1.2" + minimatch "^9.0.4" + minipass "^7.1.2" + path-scurry "^1.11.1" glob@^7.1.1, glob@^7.1.3, glob@^7.1.4: version "7.2.3" @@ -4405,18 +4349,18 @@ istextorbinary@^3.0.0: binaryextensions "^2.2.0" textextensions "^3.2.0" -ix@5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/ix/-/ix-5.0.0.tgz#b9e292f79b1876bbf696809fe86e42930bdbfcd4" - integrity sha512-6LyyrHnvNrSy5pKtW/KA+KKusHrB223aBJCJlIGPN7QBfDkEEtNrAkAz9lLLShIcdJntq6BiPCHuKaCM/9wwXw== +ix@6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/ix/-/ix-6.0.0.tgz#c1875523f8090c7146dc3ac3412a763663887f27" + integrity sha512-B/KeYkHtOWbr3ttckqWT9uha2ixw9fGVDxX+DwVXhO+P5eOhyCadt+aC30hRBvG+do+tbI3xbYDMYN6dp1C4Vw== dependencies: - "@types/node" "^13.7.4" - tslib "^2.3.0" + "@types/node" ">=13.7.4" + tslib "^2.6.2" -jackspeak@^2.3.6: - version "2.3.6" - resolved "https://registry.yarnpkg.com/jackspeak/-/jackspeak-2.3.6.tgz#647ecc472238aee4b06ac0e461acc21a8c505ca8" - integrity sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ== +jackspeak@^3.1.2: + version "3.1.2" + resolved "https://registry.yarnpkg.com/jackspeak/-/jackspeak-3.1.2.tgz#eada67ea949c6b71de50f1b09c92a961897b90ab" + integrity sha512-kWmLKn2tRtfYMF/BakihVVRzBKOxz4gJMiL2Rj91WnAB5TPZumSH99R/Yf1qE1u4uRimvCSJfm6hnxohXeEXjQ== dependencies: "@isaacs/cliui" "^8.0.2" optionalDependencies: @@ -4696,10 +4640,10 @@ jest-runtime@^29.7.0: slash "^3.0.0" strip-bom "^4.0.0" -jest-silent-reporter@0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/jest-silent-reporter/-/jest-silent-reporter-0.5.0.tgz#5fd8ccd61665227e3bf19d908b7350719d06ff38" - integrity sha512-epdLt8Oj0a1AyRiR6F8zx/1SVT1Mi7VU3y4wB2uOBHs/ohIquC7v2eeja7UN54uRPyHInIKWdL+RdG228n5pJQ== +jest-silent-reporter@0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/jest-silent-reporter/-/jest-silent-reporter-0.6.0.tgz#e9c63a3b1e3c80571d690d998faf842f576b6a60" + integrity sha512-4nmS+5o7ycVlvbQOTx7CnGdbBtP2646hnDgQpQLaVhjHcQNHD+gqBAetyjRDlgpZ8+8N82MWI59K+EX2LsVk7g== dependencies: chalk "^4.0.0" jest-util "^26.0.0" @@ -5292,13 +5236,6 @@ min-indent@^1.0.0, min-indent@^1.0.1: resolved "https://registry.yarnpkg.com/min-indent/-/min-indent-1.0.1.tgz#a63f681673b30571fbe8bc25686ae746eefa9869" integrity sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg== -minimatch@9.0.3: - version "9.0.3" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.3.tgz#a6e00c3de44c3a542bfaae70abfc22420a6da825" - integrity sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg== - dependencies: - brace-expansion "^2.0.1" - minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.1, minimatch@^3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" @@ -5306,7 +5243,7 @@ minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.1, minimatch@^3.1.2: dependencies: brace-expansion "^1.1.7" -minimatch@^9.0.1, minimatch@^9.0.3, minimatch@^9.0.4: +minimatch@^9.0.3, minimatch@^9.0.4: version "9.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.4.tgz#8e49c731d1749cbec05050ee5145147b32496a51" integrity sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw== @@ -5327,11 +5264,16 @@ minimist@1.x: resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== -"minipass@^5.0.0 || ^6.0.2 || ^7.0.0", minipass@^7.0.4: +"minipass@^5.0.0 || ^6.0.2 || ^7.0.0": version "7.0.4" resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.0.4.tgz#dbce03740f50a4786ba994c1fb908844d27b038c" integrity sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ== +minipass@^7.1.2: + version "7.1.2" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.1.2.tgz#93a9626ce5e5e66bd4db86849e7515e92340a707" + integrity sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw== + mixin-deep@^1.2.0: version "1.3.2" resolved "https://registry.yarnpkg.com/mixin-deep/-/mixin-deep-1.3.2.tgz#1120b43dc359a785dce65b55b82e257ccf479566" @@ -5722,10 +5664,10 @@ path-root@^0.1.1: dependencies: path-root-regex "^0.1.0" -path-scurry@^1.10.2: - version "1.10.2" - resolved "https://registry.yarnpkg.com/path-scurry/-/path-scurry-1.10.2.tgz#8f6357eb1239d5fa1da8b9f70e9c080675458ba7" - integrity sha512-7xTavNy5RQXnsjANvVvMkEjvloOinkAjv/Z6Ildz9v2RinZ4SBKTWFOVRbaF8p0vpHnyjV/UwNDdKuUv6M5qcA== +path-scurry@^1.11.1: + version "1.11.1" + resolved "https://registry.yarnpkg.com/path-scurry/-/path-scurry-1.11.1.tgz#7960a668888594a0720b12a911d1a742ab9f11d2" + integrity sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA== dependencies: lru-cache "^10.2.0" minipass "^5.0.0 || ^6.0.2 || ^7.0.0" @@ -6191,29 +6133,29 @@ rimraf@^3.0.2: dependencies: glob "^7.1.3" -rollup@4.17.2: - version "4.17.2" - resolved "https://registry.yarnpkg.com/rollup/-/rollup-4.17.2.tgz#26d1785d0144122277fdb20ab3a24729ae68301f" - integrity sha512-/9ClTJPByC0U4zNLowV1tMBe8yMEAxewtR3cUNX5BoEpGH3dQEWpJLr6CLp0fPdYRF/fzVOgvDb1zXuakwF5kQ== +rollup@4.18.0: + version "4.18.0" + resolved "https://registry.yarnpkg.com/rollup/-/rollup-4.18.0.tgz#497f60f0c5308e4602cf41136339fbf87d5f5dda" + integrity sha512-QmJz14PX3rzbJCN1SG4Xe/bAAX2a6NpCP8ab2vfu2GiUr8AQcr2nCV/oEO3yneFarB67zk8ShlIyWb2LGTb3Sg== dependencies: "@types/estree" "1.0.5" optionalDependencies: - "@rollup/rollup-android-arm-eabi" "4.17.2" - "@rollup/rollup-android-arm64" "4.17.2" - "@rollup/rollup-darwin-arm64" "4.17.2" - "@rollup/rollup-darwin-x64" "4.17.2" - "@rollup/rollup-linux-arm-gnueabihf" "4.17.2" - "@rollup/rollup-linux-arm-musleabihf" "4.17.2" - "@rollup/rollup-linux-arm64-gnu" "4.17.2" - "@rollup/rollup-linux-arm64-musl" "4.17.2" - "@rollup/rollup-linux-powerpc64le-gnu" "4.17.2" - "@rollup/rollup-linux-riscv64-gnu" "4.17.2" - "@rollup/rollup-linux-s390x-gnu" "4.17.2" - "@rollup/rollup-linux-x64-gnu" "4.17.2" - "@rollup/rollup-linux-x64-musl" "4.17.2" - "@rollup/rollup-win32-arm64-msvc" "4.17.2" - "@rollup/rollup-win32-ia32-msvc" "4.17.2" - "@rollup/rollup-win32-x64-msvc" "4.17.2" + "@rollup/rollup-android-arm-eabi" "4.18.0" + "@rollup/rollup-android-arm64" "4.18.0" + "@rollup/rollup-darwin-arm64" "4.18.0" + "@rollup/rollup-darwin-x64" "4.18.0" + "@rollup/rollup-linux-arm-gnueabihf" "4.18.0" + "@rollup/rollup-linux-arm-musleabihf" "4.18.0" + "@rollup/rollup-linux-arm64-gnu" "4.18.0" + "@rollup/rollup-linux-arm64-musl" "4.18.0" + "@rollup/rollup-linux-powerpc64le-gnu" "4.18.0" + "@rollup/rollup-linux-riscv64-gnu" "4.18.0" + "@rollup/rollup-linux-s390x-gnu" "4.18.0" + "@rollup/rollup-linux-x64-gnu" "4.18.0" + "@rollup/rollup-linux-x64-musl" "4.18.0" + "@rollup/rollup-win32-arm64-msvc" "4.18.0" + "@rollup/rollup-win32-ia32-msvc" "4.18.0" + "@rollup/rollup-win32-x64-msvc" "4.18.0" fsevents "~2.3.2" run-parallel@^1.1.9: @@ -6916,15 +6858,15 @@ trim-newlines@^4.0.2: resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-4.1.1.tgz#28c88deb50ed10c7ba6dc2474421904a00139125" integrity sha512-jRKj0n0jXWo6kh62nA5TEh3+4igKDXLvzBJcPpiizP7oOolUrYIxmVBG9TOtHYFHoddUk6YvAkGeGoSVTXfQXQ== -ts-api-utils@^1.0.1, ts-api-utils@^1.3.0: +ts-api-utils@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/ts-api-utils/-/ts-api-utils-1.3.0.tgz#4b490e27129f1e8e686b45cc4ab63714dc60eea1" integrity sha512-UQMIo7pb8WRomKR1/+MFVLTroIvDVtMX3K6OUir8ynLyzB8Jeriont2bTAtmNPa1ekAgN7YPDyf6V+ygrdU+eQ== -ts-jest@29.1.2: - version "29.1.2" - resolved "https://registry.yarnpkg.com/ts-jest/-/ts-jest-29.1.2.tgz#7613d8c81c43c8cb312c6904027257e814c40e09" - integrity sha512-br6GJoH/WUX4pu7FbZXuWGKGNDuU7b8Uj77g/Sp7puZV6EXzuByl6JrECvm0MzVzSTkSHWTihsXt+5XYER5b+g== +ts-jest@29.1.4: + version "29.1.4" + resolved "https://registry.yarnpkg.com/ts-jest/-/ts-jest-29.1.4.tgz#26f8a55ce31e4d2ef7a1fd47dc7fa127e92793ef" + integrity sha512-YiHwDhSvCiItoAgsKtoLFCuakDzDsJ1DLDnSouTaTmdOcOwIkSzbLXduaQ6M5DRVhuZC/NYaaZ/mtHbWMv/S6Q== dependencies: bs-logger "0.x" fast-json-stable-stringify "2.x" @@ -6954,7 +6896,7 @@ ts-node@10.9.2: v8-compile-cache-lib "^3.0.1" yn "3.1.1" -tslib@^2.0.0, tslib@^2.1.0, tslib@^2.3.0, tslib@^2.4.0, tslib@^2.6.2: +tslib@^2.0.0, tslib@^2.1.0, tslib@^2.4.0, tslib@^2.6.2: version "2.6.2" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== From 4ec1c986a033bcb286b9a0d7eb76df8e56cdc1c3 Mon Sep 17 00:00:00 2001 From: abandy Date: Mon, 3 Jun 2024 22:21:46 -0400 Subject: [PATCH 50/93] GH-41945: [Swift] Add interface ArrowArrayHolderBuilder (#41946) ### Rationale for this change This change adds the implementation of the ArrowArrayHolderBuilder interface which allows appending to Arrays and completing them without needing the generic info. This is needed for Nested types as well as for the Swift arrow Codable implementation. ### What changes are included in this PR? Adding the interface and the implementation of the interface. ### Are these changes tested? Yes, test has been added. * GitHub Issue: #41945 Authored-by: Alva Bandy Signed-off-by: Sutou Kouhei --- .../Sources/Arrow/ArrowArrayBuilder.swift | 15 +++++++++- swift/Arrow/Tests/ArrowTests/ArrayTests.swift | 28 ++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/swift/Arrow/Sources/Arrow/ArrowArrayBuilder.swift b/swift/Arrow/Sources/Arrow/ArrowArrayBuilder.swift index b78f0ccd74997..4865b8a791256 100644 --- a/swift/Arrow/Sources/Arrow/ArrowArrayBuilder.swift +++ b/swift/Arrow/Sources/Arrow/ArrowArrayBuilder.swift @@ -17,7 +17,12 @@ import Foundation -public class ArrowArrayBuilder> { +public protocol ArrowArrayHolderBuilder { + func toHolder() throws -> ArrowArrayHolder + func appendAny(_ val: Any?) +} + +public class ArrowArrayBuilder>: ArrowArrayHolderBuilder { let type: ArrowType let bufferBuilder: T public var length: UInt {return self.bufferBuilder.length} @@ -34,6 +39,10 @@ public class ArrowArrayBuilder> self.bufferBuilder.append(val) } + public func appendAny(_ val: Any?) { + self.bufferBuilder.append(val as? T.ItemType) + } + public func finish() throws -> ArrowArray { let buffers = self.bufferBuilder.finish() let arrowData = try ArrowData(self.type, buffers: buffers, nullCount: self.nullCount) @@ -43,6 +52,10 @@ public class ArrowArrayBuilder> public func getStride() -> Int { return self.type.getStride() } + + public func toHolder() throws -> ArrowArrayHolder { + return try ArrowArrayHolderImpl(self.finish()) + } } public class NumberArrayBuilder: ArrowArrayBuilder, FixedArray> { diff --git a/swift/Arrow/Tests/ArrowTests/ArrayTests.swift b/swift/Arrow/Tests/ArrowTests/ArrayTests.swift index f5bfa0506e62f..10ffc4f96d83e 100644 --- a/swift/Arrow/Tests/ArrowTests/ArrayTests.swift +++ b/swift/Arrow/Tests/ArrowTests/ArrayTests.swift @@ -245,4 +245,30 @@ final class ArrayTests: XCTestCase { try checkHolderForType(ArrowType(ArrowType.ArrowBool)) try checkHolderForType(ArrowType(ArrowType.ArrowString)) } -} + + func testArrowArrayHolderBuilder() throws { + let uint8HBuilder: ArrowArrayHolderBuilder = + (try ArrowArrayBuilders.loadNumberArrayBuilder() as NumberArrayBuilder) + for index in 0..<100 { + uint8HBuilder.appendAny(UInt8(index)) + } + + let uint8Holder = try uint8HBuilder.toHolder() + XCTAssertEqual(uint8Holder.nullCount, 0) + XCTAssertEqual(uint8Holder.length, 100) + + let stringHBuilder: ArrowArrayHolderBuilder = + (try ArrowArrayBuilders.loadStringArrayBuilder()) + for index in 0..<100 { + if index % 10 == 9 { + stringHBuilder.appendAny(nil) + } else { + stringHBuilder.appendAny("test" + String(index)) + } + } + + let stringHolder = try stringHBuilder.toHolder() + XCTAssertEqual(stringHolder.nullCount, 10) + XCTAssertEqual(stringHolder.length, 100) + } + } From 7d60148d24a454b135988a3a394938d207ad90e7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Jun 2024 14:08:15 +0900 Subject: [PATCH 51/93] MINOR: [JS] Bump @types/node from 20.13.0 to 20.14.1 in /js (#41948) Bumps [@ types/node](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/node) from 20.13.0 to 20.14.1.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=@ types/node&package-manager=npm_and_yarn&previous-version=20.13.0&new-version=20.14.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- js/yarn.lock | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/js/yarn.lock b/js/yarn.lock index d5527097340d9..c19f69a901bfa 100644 --- a/js/yarn.lock +++ b/js/yarn.lock @@ -1370,17 +1370,10 @@ dependencies: "@types/node" "*" -"@types/node@*": - version "20.12.8" - resolved "https://registry.yarnpkg.com/@types/node/-/node-20.12.8.tgz#35897bf2bfe3469847ab04634636de09552e8256" - integrity sha512-NU0rJLJnshZWdE/097cdCBbyW1h4hEg0xpovcoAQYHl8dnEyp/NAOiE45pvc+Bd1Dt+2r94v2eGFpQJ4R7g+2w== - dependencies: - undici-types "~5.26.4" - -"@types/node@>=13.7.4", "@types/node@^20.13.0": - version "20.13.0" - resolved "https://registry.yarnpkg.com/@types/node/-/node-20.13.0.tgz#011a76bc1e71ae9a026dddcfd7039084f752c4b6" - integrity sha512-FM6AOb3khNkNIXPnHFDYaHerSv8uN22C91z098AnGccVu+Pcdhi+pNUFDi0iLmPIsVE0JBD0KVS7mzUYt4nRzQ== +"@types/node@*", "@types/node@>=13.7.4", "@types/node@^20.13.0": + version "20.14.1" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.14.1.tgz#2434dbcb1f039e31f2c0e9969da93f52cf6348f3" + integrity sha512-T2MzSGEu+ysB/FkWfqmhV3PLyQlowdptmmgD20C6QxsS8Fmv5SjpZ1ayXaEC0S21/h5UJ9iA6W/5vSNU5l00OA== dependencies: undici-types "~5.26.4" From 8d2db80586a2cd65e03efc9ccd66e08e5b484692 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Jun 2024 14:08:35 +0900 Subject: [PATCH 52/93] MINOR: [JS] Bump @typescript-eslint/parser from 7.11.0 to 7.12.0 in /js (#41949) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [@ typescript-eslint/parser](https://github.com/typescript-eslint/typescript-eslint/tree/HEAD/packages/parser) from 7.11.0 to 7.12.0.
Release notes

Sourced from @​typescript-eslint/parser's releases.

v7.12.0

7.12.0 (2024-06-03)

🚀 Features

  • eslint-plugin: [no-useless-template-literals] rename to no-useless-template-expression (deprecate no-useless-template-literals) (#8821)
  • eslint-plugin: [no-floating-promises] add option 'allowForKnownSafePromises' (#9186)
  • rule-tester: check for parsing errors in suggestion fixes (#9052)
  • rule-tester: port checkDuplicateTestCases from ESLint (#9026)

🩹 Fixes

  • no-useless-template-expression -> no-unnecessary-template-expression (#9174)
  • eslint-plugin: [no-unnecessary-type-assertion] combine template literal check with const variable check (#8820)
  • eslint-plugin: [dot-notation] fix false positive when accessing private/protected property with optional chaining (#8851)
  • eslint-plugin: [explicit-member-accessibility] refine report locations (#8869)
  • eslint-plugin: [no-unnecessary-type-assertion] declares are always defined, so always check declares (#8901)
  • eslint-plugin: [prefer-literal-enum-member] allow using member it self on allowBitwiseExpressions (#9114)
  • eslint-plugin: [return-await] clean up in-try-catch detection and make autofixes safe (#9031)
  • eslint-plugin: [member-ordering] also TSMethodSignature can be get/set (#9193)
  • types: correct typing ParserOptions (#9202)

❤️ Thank You

You can read about our versioning strategy and releases on our website.

Changelog

Sourced from @​typescript-eslint/parser's changelog.

7.12.0 (2024-06-03)

🩹 Fixes

  • types: correct typing ParserOptions

❤️ Thank You

  • Abraham Guo
  • Han Yeong-woo
  • Joshua Chen
  • Kim Sang Du
  • Kirk Waiblinger
  • YeonJuan

You can read about our versioning strategy and releases on our website.

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=@ typescript-eslint/parser&package-manager=npm_and_yarn&previous-version=7.11.0&new-version=7.12.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- js/package.json | 2 +- js/yarn.lock | 53 ++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/js/package.json b/js/package.json index 9e61d94dc3b7b..b55b637e2750d 100644 --- a/js/package.json +++ b/js/package.json @@ -73,7 +73,7 @@ "@types/jest": "29.5.12", "@types/multistream": "4.1.3", "@typescript-eslint/eslint-plugin": "7.11.0", - "@typescript-eslint/parser": "7.11.0", + "@typescript-eslint/parser": "7.12.0", "async-done": "2.0.0", "benny": "3.7.1", "cross-env": "7.0.3", diff --git a/js/yarn.lock b/js/yarn.lock index c19f69a901bfa..ec311730b8918 100644 --- a/js/yarn.lock +++ b/js/yarn.lock @@ -1434,15 +1434,15 @@ natural-compare "^1.4.0" ts-api-utils "^1.3.0" -"@typescript-eslint/parser@7.11.0": - version "7.11.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-7.11.0.tgz#525ad8bee54a8f015f134edd241d91b84ab64839" - integrity sha512-yimw99teuaXVWsBcPO1Ais02kwJ1jmNA1KxE7ng0aT7ndr1pT1wqj0OJnsYVGKKlc4QJai86l/025L6z8CljOg== - dependencies: - "@typescript-eslint/scope-manager" "7.11.0" - "@typescript-eslint/types" "7.11.0" - "@typescript-eslint/typescript-estree" "7.11.0" - "@typescript-eslint/visitor-keys" "7.11.0" +"@typescript-eslint/parser@7.12.0": + version "7.12.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-7.12.0.tgz#8761df3345528b35049353db80010b385719b1c3" + integrity sha512-dm/J2UDY3oV3TKius2OUZIFHsomQmpHtsV0FTh1WO8EKgHLQ1QCADUqscPgTpU+ih1e21FQSRjXckHn3txn6kQ== + dependencies: + "@typescript-eslint/scope-manager" "7.12.0" + "@typescript-eslint/types" "7.12.0" + "@typescript-eslint/typescript-estree" "7.12.0" + "@typescript-eslint/visitor-keys" "7.12.0" debug "^4.3.4" "@typescript-eslint/scope-manager@7.11.0": @@ -1453,6 +1453,14 @@ "@typescript-eslint/types" "7.11.0" "@typescript-eslint/visitor-keys" "7.11.0" +"@typescript-eslint/scope-manager@7.12.0": + version "7.12.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-7.12.0.tgz#259c014362de72dd34f995efe6bd8dda486adf58" + integrity sha512-itF1pTnN6F3unPak+kutH9raIkL3lhH1YRPGgt7QQOh43DQKVJXmWkpb+vpc/TiDHs6RSd9CTbDsc/Y+Ygq7kg== + dependencies: + "@typescript-eslint/types" "7.12.0" + "@typescript-eslint/visitor-keys" "7.12.0" + "@typescript-eslint/type-utils@7.11.0": version "7.11.0" resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-7.11.0.tgz#ac216697d649084fedf4a910347b9642bd0ff099" @@ -1468,6 +1476,11 @@ resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-7.11.0.tgz#5e9702a5e8b424b7fc690e338d359939257d6722" integrity sha512-MPEsDRZTyCiXkD4vd3zywDCifi7tatc4K37KqTprCvaXptP7Xlpdw0NR2hRJTetG5TxbWDB79Ys4kLmHliEo/w== +"@typescript-eslint/types@7.12.0": + version "7.12.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-7.12.0.tgz#bf208f971a8da1e7524a5d9ae2b5f15192a37981" + integrity sha512-o+0Te6eWp2ppKY3mLCU+YA9pVJxhUJE15FV7kxuD9jgwIAa+w/ycGJBMrYDTpVGUM/tgpa9SeMOugSabWFq7bg== + "@typescript-eslint/typescript-estree@7.11.0": version "7.11.0" resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-7.11.0.tgz#7cbc569bc7336c3a494ceaf8204fdee5d5dbb7fa" @@ -1482,6 +1495,20 @@ semver "^7.6.0" ts-api-utils "^1.3.0" +"@typescript-eslint/typescript-estree@7.12.0": + version "7.12.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-7.12.0.tgz#e6c1074f248b3db6573ab6a7c47a39c4cd498ff9" + integrity sha512-5bwqLsWBULv1h6pn7cMW5dXX/Y2amRqLaKqsASVwbBHMZSnHqE/HN4vT4fE0aFsiwxYvr98kqOWh1a8ZKXalCQ== + dependencies: + "@typescript-eslint/types" "7.12.0" + "@typescript-eslint/visitor-keys" "7.12.0" + debug "^4.3.4" + globby "^11.1.0" + is-glob "^4.0.3" + minimatch "^9.0.4" + semver "^7.6.0" + ts-api-utils "^1.3.0" + "@typescript-eslint/utils@7.11.0", "@typescript-eslint/utils@^6.0.0 || ^7.0.0": version "7.11.0" resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-7.11.0.tgz#524f047f2209959424c3ef689b0d83b3bc09919c" @@ -1500,6 +1527,14 @@ "@typescript-eslint/types" "7.11.0" eslint-visitor-keys "^3.4.3" +"@typescript-eslint/visitor-keys@7.12.0": + version "7.12.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-7.12.0.tgz#c053b55a996679528beeedd8e565710ce1ae1ad3" + integrity sha512-uZk7DevrQLL3vSnfFl5bj4sL75qC9D6EdjemIdbtkuUmIheWpuiiylSY01JxJE7+zGrOWDZrp1WxOuDntvKrHQ== + dependencies: + "@typescript-eslint/types" "7.12.0" + eslint-visitor-keys "^3.4.3" + "@ungap/structured-clone@^1.2.0": version "1.2.0" resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.2.0.tgz#756641adb587851b5ccb3e095daf27ae581c8406" From 7bc2452b350867b3ddc9de9ceceeef0e4d722941 Mon Sep 17 00:00:00 2001 From: Vibhatha Lakmal Abeykoon Date: Tue, 4 Jun 2024 12:35:13 +0530 Subject: [PATCH 53/93] GH-41902: [Java] Variadic Buffer Counts Incorrect (#41930) ### Rationale for this change In the initial PR for `variadicBufferCounts` addition to Java spec, the non variadic buffer-ed vectors were assigned with 0 valued non-empty `variadicBufferCounts`. And this caused CIs to fail in Arrow Rust. ### What changes are included in this PR? This PR changes such that non variadic buffer-ed vectors would contain an empty `variadicBufferCounts` attribute in `ArrowRecordBatch` interface in Java. Also this includes upgrade to JUNIT5. ### Are these changes tested? Yes, from existing tests and a new test added. ### Are there any user-facing changes? No * GitHub Issue: #41902 Authored-by: Vibhatha Abeykoon Signed-off-by: David Li --- .../apache/arrow/c/StructVectorLoader.java | 29 +++++-- .../apache/arrow/c/StructVectorUnloader.java | 5 +- .../org/apache/arrow/c/DictionaryTest.java | 60 ++++++++++++++ .../org/apache/arrow/vector/VectorLoader.java | 20 +++-- .../apache/arrow/vector/VectorUnloader.java | 5 +- .../apache/arrow/vector/TestValueVector.java | 56 +++++++++++++ .../arrow/vector/TestVarCharViewVector.java | 80 +++++++++++++++++++ 7 files changed, 238 insertions(+), 17 deletions(-) diff --git a/java/c/src/main/java/org/apache/arrow/c/StructVectorLoader.java b/java/c/src/main/java/org/apache/arrow/c/StructVectorLoader.java index 27acf84d30157..1b0c59163a187 100644 --- a/java/c/src/main/java/org/apache/arrow/c/StructVectorLoader.java +++ b/java/c/src/main/java/org/apache/arrow/c/StructVectorLoader.java @@ -27,6 +27,7 @@ import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.util.Collections2; +import org.apache.arrow.vector.BaseVariableWidthViewVector; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.TypeLayout; import org.apache.arrow.vector.complex.StructVector; @@ -54,7 +55,14 @@ public class StructVectorLoader { /** * Construct with a schema. - * + *

+ * The schema referred to here can be obtained from the struct vector. + * The schema here should be the children of a struct vector, not a schema + * containing the struct field itself. + * For example: + * + * Schema schema = new Schema(structVector.getField().getChildren()); + * * @param schema buffers are added based on schema. */ public StructVectorLoader(Schema schema) { @@ -90,7 +98,7 @@ public StructVector load(BufferAllocator allocator, ArrowRecordBatch recordBatch .fromCompressionType(recordBatch.getBodyCompression().getCodec()); decompressionNeeded = codecType != CompressionUtil.CodecType.NO_COMPRESSION; CompressionCodec codec = decompressionNeeded ? factory.createCodec(codecType) : NoCompressionCodec.INSTANCE; - Iterator variadicBufferCounts = null; + Iterator variadicBufferCounts = Collections.emptyIterator(); if (recordBatch.getVariadicBufferCounts() != null && !recordBatch.getVariadicBufferCounts().isEmpty()) { variadicBufferCounts = recordBatch.getVariadicBufferCounts().iterator(); } @@ -98,9 +106,10 @@ public StructVector load(BufferAllocator allocator, ArrowRecordBatch recordBatch loadBuffers(fieldVector, fieldVector.getField(), buffers, nodes, codec, variadicBufferCounts); } result.loadFieldBuffers(new ArrowFieldNode(recordBatch.getLength(), 0), Collections.singletonList(null)); - if (nodes.hasNext() || buffers.hasNext()) { - throw new IllegalArgumentException("not all nodes and buffers were consumed. nodes: " + - Collections2.toList(nodes).toString() + " buffers: " + Collections2.toList(buffers).toString()); + if (nodes.hasNext() || buffers.hasNext() || variadicBufferCounts.hasNext()) { + throw new IllegalArgumentException("not all nodes, buffers and variadicBufferCounts were consumed. nodes: " + + Collections2.toString(nodes) + " buffers: " + Collections2.toString(buffers) + " variadicBufferCounts: " + + Collections2.toString(variadicBufferCounts)); } return result; } @@ -109,10 +118,14 @@ private void loadBuffers(FieldVector vector, Field field, Iterator buf CompressionCodec codec, Iterator variadicBufferCounts) { checkArgument(nodes.hasNext(), "no more field nodes for field %s and vector %s", field, vector); ArrowFieldNode fieldNode = nodes.next(); - // variadicBufferLayoutCount will be 0 for vectors of type except BaseVariableWidthViewVector + // variadicBufferLayoutCount will be 0 for vectors of a type except BaseVariableWidthViewVector long variadicBufferLayoutCount = 0; - if (variadicBufferCounts != null) { - variadicBufferLayoutCount = variadicBufferCounts.next(); + if (vector instanceof BaseVariableWidthViewVector) { + if (variadicBufferCounts.hasNext()) { + variadicBufferLayoutCount = variadicBufferCounts.next(); + } else { + throw new IllegalStateException("No variadicBufferCounts available for BaseVariableWidthViewVector"); + } } int bufferLayoutCount = (int) (variadicBufferLayoutCount + TypeLayout.getTypeBufferCount(field.getType())); List ownBuffers = new ArrayList<>(bufferLayoutCount); diff --git a/java/c/src/main/java/org/apache/arrow/c/StructVectorUnloader.java b/java/c/src/main/java/org/apache/arrow/c/StructVectorUnloader.java index 8d015157ebf38..82539acf6f292 100644 --- a/java/c/src/main/java/org/apache/arrow/c/StructVectorUnloader.java +++ b/java/c/src/main/java/org/apache/arrow/c/StructVectorUnloader.java @@ -109,7 +109,10 @@ private void appendNodes(FieldVector vector, List nodes, List fieldBuffers = vector.getFieldBuffers(); long variadicBufferCount = getVariadicBufferCount(vector); int expectedBufferCount = (int) (TypeLayout.getTypeBufferCount(vector.getField().getType()) + variadicBufferCount); - variadicBufferCounts.add(variadicBufferCount); + // only update variadicBufferCounts for vectors that have variadic buffers + if (variadicBufferCount > 0) { + variadicBufferCounts.add(variadicBufferCount); + } if (fieldBuffers.size() != expectedBufferCount) { throw new IllegalArgumentException(String.format("wrong number of buffers for field %s in vector %s. found: %s", vector.getField(), vector.getClass().getSimpleName(), fieldBuffers)); diff --git a/java/c/src/test/java/org/apache/arrow/c/DictionaryTest.java b/java/c/src/test/java/org/apache/arrow/c/DictionaryTest.java index d892781756ede..aa1264e4842eb 100644 --- a/java/c/src/test/java/org/apache/arrow/c/DictionaryTest.java +++ b/java/c/src/test/java/org/apache/arrow/c/DictionaryTest.java @@ -17,6 +17,8 @@ package org.apache.arrow.c; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayInputStream; @@ -29,6 +31,7 @@ import org.apache.arrow.c.ArrowSchema; import org.apache.arrow.c.CDataDictionaryProvider; import org.apache.arrow.c.Data; +import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.util.AutoCloseables; import org.apache.arrow.vector.FieldVector; @@ -36,13 +39,19 @@ import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ViewVarCharVector; import org.apache.arrow.vector.compare.VectorEqualsVisitor; +import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.dictionary.Dictionary; import org.apache.arrow.vector.dictionary.DictionaryEncoder; import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.ipc.ArrowStreamReader; import org.apache.arrow.vector.ipc.ArrowStreamWriter; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; +import org.apache.arrow.vector.types.Types.MinorType; import org.apache.arrow.vector.types.pojo.DictionaryEncoding; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -216,4 +225,55 @@ private ArrowStreamReader createMultiBatchReader() throws IOException { return new ArrowStreamReader(in, allocator); } + private void createStructVector(StructVector vector) { + final ViewVarCharVector child1 = vector.addOrGet("f0", + FieldType.nullable(MinorType.VIEWVARCHAR.getType()), ViewVarCharVector.class); + final IntVector child2 = vector.addOrGet("f1", + FieldType.nullable(MinorType.INT.getType()), IntVector.class); + + // Write the values to child 1 + child1.allocateNew(); + child1.set(0, "01234567890".getBytes()); + child1.set(1, "012345678901234567".getBytes()); + vector.setIndexDefined(0); + + // Write the values to child 2 + child2.allocateNew(); + child2.set(0, 10); + child2.set(1, 11); + vector.setIndexDefined(1); + + vector.setValueCount(2); + } + + @Test + public void testVectorLoadUnloadOnStructVector() { + try (final StructVector structVector1 = StructVector.empty("struct", allocator)) { + createStructVector(structVector1); + Field field1 = structVector1.getField(); + Schema schema = new Schema(field1.getChildren()); + StructVectorUnloader vectorUnloader = new StructVectorUnloader(structVector1); + + try ( + ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch(); + BufferAllocator finalVectorsAllocator = allocator.newChildAllocator("struct", 0, Long.MAX_VALUE); + ) { + // validating recordBatch contains an output for variadicBufferCounts + assertFalse(recordBatch.getVariadicBufferCounts().isEmpty()); + assertEquals(1, recordBatch.getVariadicBufferCounts().size()); + assertEquals(1, recordBatch.getVariadicBufferCounts().get(0)); + + StructVectorLoader vectorLoader = new StructVectorLoader(schema); + try (StructVector structVector2 = vectorLoader.load(finalVectorsAllocator, recordBatch)) { + // Improve this after fixing https://github.com/apache/arrow/issues/41933 + // assertTrue(VectorEqualsVisitor.vectorEquals(structVector1, structVector2), "vectors are not equivalent"); + assertTrue(VectorEqualsVisitor.vectorEquals(structVector1.getChild("f0"), structVector2.getChild("f0")), + "vectors are not equivalent"); + assertTrue(VectorEqualsVisitor.vectorEquals(structVector1.getChild("f1"), structVector2.getChild("f1")), + "vectors are not equivalent"); + } + } + } + } + } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java index 9590e70f46770..dec536ae6cc1f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java @@ -20,6 +20,7 @@ import static org.apache.arrow.util.Preconditions.checkArgument; import java.util.ArrayList; +import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -80,7 +81,7 @@ public void load(ArrowRecordBatch recordBatch) { CompressionUtil.CodecType.fromCompressionType(recordBatch.getBodyCompression().getCodec()); decompressionNeeded = codecType != CompressionUtil.CodecType.NO_COMPRESSION; CompressionCodec codec = decompressionNeeded ? factory.createCodec(codecType) : NoCompressionCodec.INSTANCE; - Iterator variadicBufferCounts = null; + Iterator variadicBufferCounts = Collections.emptyIterator();; if (recordBatch.getVariadicBufferCounts() != null && !recordBatch.getVariadicBufferCounts().isEmpty()) { variadicBufferCounts = recordBatch.getVariadicBufferCounts().iterator(); } @@ -89,9 +90,10 @@ public void load(ArrowRecordBatch recordBatch) { loadBuffers(fieldVector, fieldVector.getField(), buffers, nodes, codec, variadicBufferCounts); } root.setRowCount(recordBatch.getLength()); - if (nodes.hasNext() || buffers.hasNext()) { - throw new IllegalArgumentException("not all nodes and buffers were consumed. nodes: " + - Collections2.toString(nodes) + " buffers: " + Collections2.toString(buffers)); + if (nodes.hasNext() || buffers.hasNext() || variadicBufferCounts.hasNext()) { + throw new IllegalArgumentException("not all nodes, buffers and variadicBufferCounts were consumed. nodes: " + + Collections2.toString(nodes) + " buffers: " + Collections2.toString(buffers) + " variadicBufferCounts: " + + Collections2.toString(variadicBufferCounts)); } } @@ -104,10 +106,14 @@ private void loadBuffers( Iterator variadicBufferCounts) { checkArgument(nodes.hasNext(), "no more field nodes for field %s and vector %s", field, vector); ArrowFieldNode fieldNode = nodes.next(); - // variadicBufferLayoutCount will be 0 for vectors of type except BaseVariableWidthViewVector + // variadicBufferLayoutCount will be 0 for vectors of a type except BaseVariableWidthViewVector long variadicBufferLayoutCount = 0; - if (variadicBufferCounts != null) { - variadicBufferLayoutCount = variadicBufferCounts.next(); + if (vector instanceof BaseVariableWidthViewVector) { + if (variadicBufferCounts.hasNext()) { + variadicBufferLayoutCount = variadicBufferCounts.next(); + } else { + throw new IllegalStateException("No variadicBufferCounts available for BaseVariableWidthViewVector"); + } } int bufferLayoutCount = (int) (variadicBufferLayoutCount + TypeLayout.getTypeBufferCount(field.getType())); List ownBuffers = new ArrayList<>(bufferLayoutCount); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java index 8528099b6d619..6e7ab34eba9de 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java @@ -103,7 +103,10 @@ private void appendNodes(FieldVector vector, List nodes, List fieldBuffers = vector.getFieldBuffers(); long variadicBufferCount = getVariadicBufferCount(vector); int expectedBufferCount = (int) (TypeLayout.getTypeBufferCount(vector.getField().getType()) + variadicBufferCount); - variadicBufferCounts.add(variadicBufferCount); + // only update variadicBufferCounts for vectors that have variadic buffers + if (variadicBufferCount > 0) { + variadicBufferCounts.add(variadicBufferCount); + } if (fieldBuffers.size() != expectedBufferCount) { throw new IllegalArgumentException(String.format( "wrong number of buffers for field %s in vector %s. found: %s", diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index fda14b24a4c8b..b0d316070a335 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -3441,4 +3441,60 @@ public void testSplitAndTransferFixedWithVector2() { } target.close(); } + + @Test + public void testVectorLoadUnloadOnNonVariadicVectors() { + + try (final IntVector vector1 = new IntVector("myvector", allocator)) { + + setVector(vector1, 1, 2, 3, 4, 5, 6); + vector1.setValueCount(15); + + /* Check the vector output */ + assertEquals(1, vector1.get(0)); + assertEquals(2, vector1.get(1)); + assertEquals(3, vector1.get(2)); + assertEquals(4, vector1.get(3)); + assertEquals(5, vector1.get(4)); + assertEquals(6, vector1.get(5)); + + Field field = vector1.getField(); + String fieldName = field.getName(); + + List fields = new ArrayList<>(); + List fieldVectors = new ArrayList<>(); + + fields.add(field); + fieldVectors.add(vector1); + + Schema schema = new Schema(fields); + + VectorSchemaRoot schemaRoot1 = new VectorSchemaRoot(schema, fieldVectors, vector1.getValueCount()); + VectorUnloader vectorUnloader = new VectorUnloader(schemaRoot1); + + try ( + ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch(); + BufferAllocator finalVectorsAllocator = allocator.newChildAllocator("new vector", 0, Long.MAX_VALUE); + VectorSchemaRoot schemaRoot2 = VectorSchemaRoot.create(schema, finalVectorsAllocator); + ) { + + // validating recordBatch doesn't contain an output for variadicBufferCounts + assertTrue(recordBatch.getVariadicBufferCounts().isEmpty()); + + VectorLoader vectorLoader = new VectorLoader(schemaRoot2); + vectorLoader.load(recordBatch); + + IntVector vector2 = (IntVector) schemaRoot2.getVector(fieldName); + vector2.setValueCount(25); + + /* Check the vector output */ + assertEquals(1, vector2.get(0)); + assertEquals(2, vector2.get(1)); + assertEquals(3, vector2.get(2)); + assertEquals(4, vector2.get(3)); + assertEquals(5, vector2.get(4)); + assertEquals(6, vector2.get(5)); + } + } + } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharViewVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharViewVector.java index 1ba3bc3576fb2..817941ecb46d6 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharViewVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharViewVector.java @@ -23,6 +23,7 @@ import static org.apache.arrow.vector.testing.ValueVectorDataPopulator.setVector; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; @@ -2212,6 +2213,85 @@ public void testSplitAndTransfer9() { } } + @Test + public void testVectorLoadUnloadOnMixedTypes() { + + try (final IntVector vector1 = new IntVector("myvector", allocator); + final ViewVarCharVector vector2 = new ViewVarCharVector("myviewvector", allocator)) { + + final int valueCount = 15; + + setVector(vector1, 1, 2, 3, 4, 5, 6); + vector1.setValueCount(valueCount); + + setVector(vector2, STR1, STR2, STR3, STR4, STR5, STR6); + vector1.setValueCount(valueCount); + + /* Check the vector output */ + assertEquals(1, vector1.get(0)); + assertEquals(2, vector1.get(1)); + assertEquals(3, vector1.get(2)); + assertEquals(4, vector1.get(3)); + assertEquals(5, vector1.get(4)); + assertEquals(6, vector1.get(5)); + + Field field1 = vector1.getField(); + String fieldName1 = field1.getName(); + + Field field2 = vector2.getField(); + String fieldName2 = field2.getName(); + + List fields = new ArrayList<>(2); + List fieldVectors = new ArrayList<>(2); + + fields.add(field1); + fields.add(field2); + fieldVectors.add(vector1); + fieldVectors.add(vector2); + + Schema schema = new Schema(fields); + + VectorSchemaRoot schemaRoot1 = new VectorSchemaRoot(schema, fieldVectors, valueCount); + VectorUnloader vectorUnloader = new VectorUnloader(schemaRoot1); + + try ( + ArrowRecordBatch recordBatch = vectorUnloader.getRecordBatch(); + BufferAllocator finalVectorsAllocator = allocator.newChildAllocator("new vector", 0, Long.MAX_VALUE); + VectorSchemaRoot schemaRoot2 = VectorSchemaRoot.create(schema, finalVectorsAllocator); + ) { + + // validating recordBatch contains an output for variadicBufferCounts + assertFalse(recordBatch.getVariadicBufferCounts().isEmpty()); + assertEquals(1, recordBatch.getVariadicBufferCounts().size()); + + VectorLoader vectorLoader = new VectorLoader(schemaRoot2); + vectorLoader.load(recordBatch); + + IntVector vector3 = (IntVector) schemaRoot2.getVector(fieldName1); + vector3.setValueCount(25); + + /* Check the vector output */ + assertEquals(1, vector3.get(0)); + assertEquals(2, vector3.get(1)); + assertEquals(3, vector3.get(2)); + assertEquals(4, vector3.get(3)); + assertEquals(5, vector3.get(4)); + assertEquals(6, vector3.get(5)); + + ViewVarCharVector vector4 = (ViewVarCharVector) schemaRoot2.getVector(fieldName2); + vector4.setValueCount(25); + + /* Check the vector output */ + assertArrayEquals(STR1, vector4.get(0)); + assertArrayEquals(STR2, vector4.get(1)); + assertArrayEquals(STR3, vector4.get(2)); + assertArrayEquals(STR4, vector4.get(3)); + assertArrayEquals(STR5, vector4.get(4)); + assertArrayEquals(STR6, vector4.get(5)); + } + } + } + private String generateRandomString(int length) { Random random = new Random(); StringBuilder sb = new StringBuilder(length); From 524a463207fdb799b2cd784b7ef95052165882ec Mon Sep 17 00:00:00 2001 From: Vibhatha Lakmal Abeykoon Date: Tue, 4 Jun 2024 14:25:05 +0530 Subject: [PATCH 54/93] GH-39649: [Java][CI] Fix or suppress spurious errorprone warnings stage 2 (#39777) ### Rationale for this change This PR is a continuation of errorprone warning fixes. ### What changes are included in this PR? Fixing warnings suggested by the errorpone module. - [x] Adapter - [x] C - [x] Format - [x] Maven - [x] Memory - [x] Performance ### Are these changes tested? The existing test cases will be covering them. ### Are there any user-facing changes? No * Closes: #39649 Lead-authored-by: Vibhatha Abeykoon Co-authored-by: Vibhatha Lakmal Abeykoon Co-authored-by: vibhatha Co-authored-by: Vibhatha Lakmal Abeykoon Co-authored-by: David Li Signed-off-by: David Li --- docs/source/developers/java/development.rst | 16 +++++++++++ .../arrow/adapter/avro/AvroToArrowUtils.java | 28 +++++++++---------- .../avro/AvroToArrowVectorIterator.java | 2 ++ .../adapter/avro/consumers/Consumer.java | 1 + .../arrow/adapter/avro/AvroSkipFieldTest.java | 9 +++--- .../arrow/adapter/avro/AvroTestBase.java | 4 +-- .../adapter/avro/AvroToArrowIteratorTest.java | 4 +-- .../arrow/adapter/avro/AvroToArrowTest.java | 2 -- .../adapter/jdbc/consumer/BinaryConsumer.java | 2 +- .../adapter/jdbc/consumer/ClobConsumer.java | 8 +++--- .../adapter/jdbc/consumer/JdbcConsumer.java | 1 + .../adapter/jdbc/JdbcParameterBinderTest.java | 10 +++---- .../jdbc/JdbcToArrowCommentMetadataTest.java | 9 ------ .../adapter/jdbc/JdbcToArrowConfigTest.java | 8 +++--- .../adapter/jdbc/JdbcToArrowTestHelper.java | 7 +++-- .../arrow/adapter/jdbc/ResultSetUtility.java | 23 ++++++--------- .../adapter/jdbc/UnreliableMetaDataTest.java | 3 -- .../adapter/jdbc/h2/JdbcAliasToArrowTest.java | 4 +-- .../adapter/jdbc/h2/JdbcToArrowArrayTest.java | 3 +- .../jdbc/h2/JdbcToArrowCharSetTest.java | 3 ++ .../jdbc/h2/JdbcToArrowDataTypesTest.java | 2 ++ .../jdbc/h2/JdbcToArrowMapDataTypeTest.java | 2 ++ .../adapter/jdbc/h2/JdbcToArrowNullTest.java | 2 ++ .../h2/JdbcToArrowOptionalColumnsTest.java | 2 ++ .../adapter/jdbc/h2/JdbcToArrowTest.java | 2 ++ .../jdbc/h2/JdbcToArrowTimeZoneTest.java | 2 ++ .../main/java/org/apache/arrow/c/Format.java | 3 +- .../apache/arrow/c/jni/CDataJniException.java | 4 +-- .../org/apache/arrow/c/DictionaryTest.java | 25 +++++++++-------- .../org/apache/arrow/c/NativeUtilTest.java | 2 +- .../org/apache/arrow/c/RoundtripTest.java | 4 +-- .../memory/AllocationOutcomeDetails.java | 18 ++++++------ .../arrow/memory/AllocationReservation.java | 2 +- .../apache/arrow/memory/BaseAllocator.java | 2 ++ .../org/apache/arrow/memory/BufferLedger.java | 3 +- .../apache/arrow/memory/BufferManager.java | 2 +- .../apache/arrow/memory/CheckAllocator.java | 1 + .../apache/arrow/memory/ChildAllocator.java | 2 +- .../arrow/memory/LowCostIdentityHashMap.java | 3 +- .../rounding/DefaultRoundingPolicy.java | 4 --- .../arrow/memory/util/ArrowBufPointer.java | 3 +- .../org/apache/arrow/memory/util/Float16.java | 4 +-- .../arrow/memory/util/HistoricalLog.java | 11 ++++++-- .../apache/arrow/memory/util/MemoryUtil.java | 1 + .../arrow/memory/util/hash/MurmurHasher.java | 2 +- .../arrow/memory/util/hash/SimpleHasher.java | 1 + .../org/apache/arrow/memory/TestArrowBuf.java | 2 +- .../arrow/memory/TestBaseAllocator.java | 4 +-- .../memory/TestLowCostIdentityHashMap.java | 2 +- .../memory/util/TestArrowBufPointer.java | 20 ++++++------- .../memory/util/TestByteFunctionHelpers.java | 6 ++-- .../memory/util/hash/TestArrowBufHasher.java | 12 ++++---- .../java/io/netty/buffer/NettyArrowBuf.java | 2 +- .../buffer/UnsafeDirectLittleEndian.java | 19 ++++--------- .../buffer/TestUnsafeDirectLittleEndian.java | 3 +- .../memory/netty/NettyAllocationManager.java | 4 +-- .../arrow/memory/netty/TestEndianness.java | 16 +++++------ .../memory/netty/TestNettyAllocator.java | 1 + java/pom.xml | 4 ++- 59 files changed, 187 insertions(+), 164 deletions(-) diff --git a/docs/source/developers/java/development.rst b/docs/source/developers/java/development.rst index 3f0ff6cdd0103..9f78eccf6c525 100644 --- a/docs/source/developers/java/development.rst +++ b/docs/source/developers/java/development.rst @@ -137,3 +137,19 @@ This applies the style to all pom.xml files under the current directory or from .. _conbench: https://github.com/conbench/conbench .. _checkstyle: https://github.com/apache/arrow/blob/main/java/dev/checkstyle/checkstyle.xml .. _Apache Maven pom.xml guidelines: https://maven.apache.org/developers/conventions/code.html#pom-code-convention + + +Build Caching +============= + +Build caching is done through Develocity (formerly Maven Enterprise). To force +a build without the cache, run:: + + mvn clean install -Ddevelocity.cache.local.enabled=false -Ddevelocity.cache.remote.enabled=false + +This can be useful to make sure you see all warnings from ErrorProne, for example. + +ErrorProne +========== + +ErrorProne should be disabled for generated code. diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java index 1f5ad9e768950..c949f4b1ec5b6 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowUtils.java @@ -27,6 +27,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -95,7 +96,6 @@ import org.apache.avro.LogicalType; import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; -import org.apache.avro.Schema.Type; import org.apache.avro.io.Decoder; /** @@ -159,7 +159,7 @@ private static Consumer createConsumer( final BufferAllocator allocator = config.getAllocator(); - final Type type = schema.getType(); + final Schema.Type type = schema.getType(); final LogicalType logicalType = schema.getLogicalType(); final ArrowType arrowType; @@ -215,7 +215,7 @@ private static Consumer createConsumer( vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroTimeMillisConsumer((TimeMilliVector) vector); } else { - arrowType = new ArrowType.Int(32, /*signed=*/true); + arrowType = new ArrowType.Int(32, /*isSigned=*/true); fieldType = new FieldType(nullable, arrowType, /*dictionary=*/null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroIntConsumer((IntVector) vector); @@ -244,7 +244,7 @@ private static Consumer createConsumer( vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroTimestampMicrosConsumer((TimeStampMicroVector) vector); } else { - arrowType = new ArrowType.Int(64, /*signed=*/true); + arrowType = new ArrowType.Int(64, /*isSigned=*/true); fieldType = new FieldType(nullable, arrowType, /*dictionary=*/null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroLongConsumer((BigIntVector) vector); @@ -278,7 +278,7 @@ private static Consumer createConsumer( case NULL: arrowType = new ArrowType.Null(); fieldType = new FieldType(nullable, arrowType, /*dictionary=*/null, getMetaData(schema)); - vector = fieldType.createNewSingleVector(name, allocator, /*schemaCallback=*/null); + vector = fieldType.createNewSingleVector(name, allocator, /*schemaCallBack=*/null); consumer = new AvroNullConsumer((NullVector) vector); break; default: @@ -305,7 +305,7 @@ private static ArrowType createDecimalArrowType(LogicalTypes.Decimal logicalType private static Consumer createSkipConsumer(Schema schema) { SkipFunction skipFunction; - Type type = schema.getType(); + Schema.Type type = schema.getType(); switch (type) { case UNION: @@ -391,7 +391,7 @@ static CompositeAvroConsumer createCompositeConsumer( final Set skipFieldNames = config.getSkipFieldNames(); Schema.Type type = schema.getType(); - if (type == Type.RECORD) { + if (type == Schema.Type.RECORD) { for (Schema.Field field : schema.getFields()) { if (skipFieldNames.contains(field.name())) { consumers.add(createSkipConsumer(field.schema())); @@ -416,7 +416,7 @@ private static FieldVector createVector(FieldVector consumerVector, FieldType fi private static String getDefaultFieldName(ArrowType type) { Types.MinorType minorType = Types.getMinorTypeForArrowType(type); - return minorType.name().toLowerCase(); + return minorType.name().toLowerCase(Locale.ROOT); } private static Field avroSchemaToField(Schema schema, String name, AvroToArrowConfig config) { @@ -429,7 +429,7 @@ private static Field avroSchemaToField( AvroToArrowConfig config, Map externalProps) { - final Type type = schema.getType(); + final Schema.Type type = schema.getType(); final LogicalType logicalType = schema.getLogicalType(); final List children = new ArrayList<>(); final FieldType fieldType; @@ -457,7 +457,7 @@ private static Field avroSchemaToField( FieldType structFieldType = new FieldType(false, new ArrowType.Struct(), /*dictionary=*/null); Field structField = new Field("internal", structFieldType, Arrays.asList(keyField, valueField)); children.add(structField); - fieldType = createFieldType(new ArrowType.Map(/*keySorted=*/false), schema, externalProps); + fieldType = createFieldType(new ArrowType.Map(/*keysSorted=*/false), schema, externalProps); break; case RECORD: final Set skipFieldNames = config.getSkipFieldNames(); @@ -509,7 +509,7 @@ private static Field avroSchemaToField( } else if (logicalType instanceof LogicalTypes.TimeMillis) { intArrowType = new ArrowType.Time(TimeUnit.MILLISECOND, 32); } else { - intArrowType = new ArrowType.Int(32, /*signed=*/true); + intArrowType = new ArrowType.Int(32, /*isSigned=*/true); } fieldType = createFieldType(intArrowType, schema, externalProps); break; @@ -525,7 +525,7 @@ private static Field avroSchemaToField( } else if (logicalType instanceof LogicalTypes.TimestampMicros) { longArrowType = new ArrowType.Timestamp(TimeUnit.MICROSECOND, null); } else { - longArrowType = new ArrowType.Int(64, /*signed=*/true); + longArrowType = new ArrowType.Int(64, /*isSigned=*/true); } fieldType = createFieldType(longArrowType, schema, externalProps); break; @@ -668,7 +668,7 @@ private static Consumer createUnionConsumer(Schema schema, String name, AvroToAr FieldVector consumerVector) { final int size = schema.getTypes().size(); - final boolean nullable = schema.getTypes().stream().anyMatch(t -> t.getType() == Type.NULL); + final boolean nullable = schema.getTypes().stream().anyMatch(t -> t.getType() == Schema.Type.NULL); UnionVector unionVector; if (consumerVector == null) { @@ -709,7 +709,7 @@ static VectorSchemaRoot avroToArrowVectors( final Set skipFieldNames = config.getSkipFieldNames(); Schema.Type type = schema.getType(); - if (type == Type.RECORD) { + if (type == Schema.Type.RECORD) { for (Schema.Field field : schema.getFields()) { if (skipFieldNames.contains(field.name())) { consumers.add(createSkipConsumer(field.schema())); diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowVectorIterator.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowVectorIterator.java index 4a439ade81181..9a0cfd97a49a1 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowVectorIterator.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowVectorIterator.java @@ -162,6 +162,7 @@ public boolean hasNext() { /** * Gets the next vector. The user is responsible for freeing its resources. */ + @Override public VectorSchemaRoot next() { Preconditions.checkArgument(hasNext()); VectorSchemaRoot returned = nextBatch; @@ -177,6 +178,7 @@ public VectorSchemaRoot next() { /** * Clean up resources. */ + @Override public void close() { if (nextBatch != null) { nextBatch.close(); diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/Consumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/Consumer.java index c2ae1ce77b282..8eaaf74cff68a 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/Consumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/Consumer.java @@ -53,6 +53,7 @@ public interface Consumer extends AutoCloseable { /** * Close this consumer when occurs exception to avoid potential leak. */ + @Override void close() throws Exception; /** diff --git a/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroSkipFieldTest.java b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroSkipFieldTest.java index a37eca6514e04..54fa26afe3fa8 100644 --- a/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroSkipFieldTest.java +++ b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroSkipFieldTest.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -223,7 +224,7 @@ public void testSkipStringField() throws Exception { ArrayList expectedData = new ArrayList<>(); for (int i = 0; i < 5; i++) { - final byte[] testBytes = ("test" + i).getBytes(); + final byte[] testBytes = ("test" + i).getBytes(StandardCharsets.UTF_8); GenericRecord record = new GenericData.Record(schema); GenericData.Fixed fixed = new GenericData.Fixed(schema.getField("f0").schema()); fixed.bytes(testBytes); @@ -257,7 +258,7 @@ public void testSkipBytesField() throws Exception { ArrayList expectedData = new ArrayList<>(); for (int i = 0; i < 5; i++) { - final byte[] testBytes = ("test" + i).getBytes(); + final byte[] testBytes = ("test" + i).getBytes(StandardCharsets.UTF_8); GenericRecord record = new GenericData.Record(schema); GenericData.Fixed fixed = new GenericData.Fixed(schema.getField("f0").schema()); fixed.bytes(testBytes); @@ -291,7 +292,7 @@ public void testSkipFixedField() throws Exception { ArrayList expectedData = new ArrayList<>(); for (int i = 0; i < 5; i++) { - final byte[] testBytes = ("test" + i).getBytes(); + final byte[] testBytes = ("test" + i).getBytes(StandardCharsets.UTF_8); GenericRecord record = new GenericData.Record(schema); GenericData.Fixed fixed = new GenericData.Fixed(schema.getField("f0").schema()); fixed.bytes(testBytes); @@ -325,7 +326,7 @@ public void testSkipEnumField() throws Exception { ArrayList expectedData = new ArrayList<>(); for (int i = 0; i < 5; i++) { - final byte[] testBytes = ("test" + i).getBytes(); + final byte[] testBytes = ("test" + i).getBytes(StandardCharsets.UTF_8); GenericRecord record = new GenericData.Record(schema); GenericData.Fixed fixed = new GenericData.Fixed(schema.getField("f0").schema()); fixed.bytes(testBytes); diff --git a/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroTestBase.java b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroTestBase.java index 60a3a285db3aa..a91bba7b84fb4 100644 --- a/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroTestBase.java +++ b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroTestBase.java @@ -145,7 +145,7 @@ protected void checkPrimitiveResult(List data, FieldVector vector) { } } - protected void checkRecordResult(Schema schema, ArrayList data, VectorSchemaRoot root) { + protected void checkRecordResult(Schema schema, List data, VectorSchemaRoot root) { assertEquals(data.size(), root.getRowCount()); assertEquals(schema.getFields().size(), root.getFieldVectors().size()); @@ -194,7 +194,7 @@ protected void checkArrayResult(List> expected, List vectors } } - protected void checkRecordResult(Schema schema, ArrayList data, List roots) { + protected void checkRecordResult(Schema schema, List data, List roots) { roots.forEach(root -> { assertEquals(schema.getFields().size(), root.getFieldVectors().size()); }); diff --git a/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowIteratorTest.java b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowIteratorTest.java index 02f7a3733734c..7f2edb08fdabc 100644 --- a/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowIteratorTest.java +++ b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowIteratorTest.java @@ -181,13 +181,13 @@ public void runLargeNumberOfRows() throws Exception { } } - assertEquals(x, targetRows); + assertEquals(targetRows, x); } /** * Fake avro decoder to test large data. */ - private class FakeDecoder extends Decoder { + private static class FakeDecoder extends Decoder { private int numRows; diff --git a/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowTest.java b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowTest.java index 1c64204191762..26f72173b6b7e 100644 --- a/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowTest.java +++ b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowTest.java @@ -87,10 +87,8 @@ public void testFixedAttributes() throws Exception { Schema schema = getSchema("attrs/test_fixed_attr.avsc"); List data = new ArrayList<>(); - List expected = new ArrayList<>(); for (int i = 0; i < 5; i++) { byte[] value = ("value" + i).getBytes(StandardCharsets.UTF_8); - expected.add(value); GenericData.Fixed fixed = new GenericData.Fixed(schema); fixed.bytes(value); data.add(fixed); diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumer.java index 8c5f61169d405..538d161f9e9c7 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumer.java @@ -74,7 +74,7 @@ public void consume(InputStream is) throws IOException { vector.getDataBuffer().setBytes(startOffset + dataLength, reuseBytes, 0, read); dataLength += read; } - offsetBuffer.setInt((currentIndex + 1) * VarBinaryVector.OFFSET_WIDTH, startOffset + dataLength); + offsetBuffer.setInt((currentIndex + 1) * ((long) VarBinaryVector.OFFSET_WIDTH), startOffset + dataLength); BitVectorHelper.setBit(vector.getValidityBuffer(), currentIndex); vector.setLastSet(currentIndex); } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/ClobConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/ClobConsumer.java index a52d9b73b4db0..3ed0c2d3cbb2f 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/ClobConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/ClobConsumer.java @@ -88,7 +88,7 @@ public void consume(ResultSet resultSet) throws SQLException { ArrowBuf dataBuffer = vector.getDataBuffer(); ArrowBuf offsetBuffer = vector.getOffsetBuffer(); - int startIndex = offsetBuffer.getInt(currentIndex * 4); + int startIndex = offsetBuffer.getInt(currentIndex * 4L); while (read <= length) { String str = clob.getSubString(read, readSize); byte[] bytes = str.getBytes(StandardCharsets.UTF_8); @@ -106,7 +106,7 @@ public void consume(ResultSet resultSet) throws SQLException { totalBytes += bytes.length; read += readSize; } - offsetBuffer.setInt((currentIndex + 1) * 4, startIndex + totalBytes); + offsetBuffer.setInt((currentIndex + 1) * 4L, startIndex + totalBytes); BitVectorHelper.setBit(vector.getValidityBuffer(), currentIndex); vector.setLastSet(currentIndex); } @@ -139,7 +139,7 @@ public void consume(ResultSet resultSet) throws SQLException { ArrowBuf dataBuffer = vector.getDataBuffer(); ArrowBuf offsetBuffer = vector.getOffsetBuffer(); - int startIndex = offsetBuffer.getInt(currentIndex * 4); + int startIndex = offsetBuffer.getInt(currentIndex * 4L); while (read <= length) { String str = clob.getSubString(read, readSize); byte[] bytes = str.getBytes(StandardCharsets.UTF_8); @@ -157,7 +157,7 @@ public void consume(ResultSet resultSet) throws SQLException { totalBytes += bytes.length; read += readSize; } - offsetBuffer.setInt((currentIndex + 1) * 4, startIndex + totalBytes); + offsetBuffer.setInt((currentIndex + 1) * 4L, startIndex + totalBytes); BitVectorHelper.setBit(vector.getValidityBuffer(), currentIndex); vector.setLastSet(currentIndex); } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/JdbcConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/JdbcConsumer.java index 480dfe3a1c57f..7c867c7ad64d3 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/JdbcConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/JdbcConsumer.java @@ -37,6 +37,7 @@ public interface JdbcConsumer extends AutoCloseable { /** * Close this consumer, do some clean work such as clear reuse ArrowBuf. */ + @Override void close() throws Exception; /** diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcParameterBinderTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcParameterBinderTest.java index 15b9ab0386159..a94f0aa454f1d 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcParameterBinderTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcParameterBinderTest.java @@ -109,8 +109,8 @@ void bindOrder() throws SQLException { final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { final JdbcParameterBinder binder = JdbcParameterBinder.builder(statement, root) - .bind(/*paramIndex=*/ 1, /*colIndex=*/ 2) - .bind(/*paramIndex=*/ 2, /*colIndex=*/ 0) + .bind(/*parameterIndex=*/ 1, /*columnIndex=*/ 2) + .bind(/*parameterIndex=*/ 2, /*columnIndex=*/ 0) .build(); assertThat(binder.next()).isFalse(); @@ -169,7 +169,7 @@ void customBinder() throws SQLException { final JdbcParameterBinder binder = JdbcParameterBinder.builder(statement, root) .bind( - /*paramIndex=*/ 1, + /*parameterIndex=*/ 1, new ColumnBinder() { private final IntVector vector = (IntVector) root.getVector(0); @Override @@ -275,11 +275,11 @@ void time32() throws SQLException { @Test void time64() throws SQLException { testSimpleType(new ArrowType.Time(TimeUnit.MICROSECOND, 64), Types.TIME, - (valueVectors, index, value) -> valueVectors.setSafe(index, (int) (value.getTime() * 1_000)), + (valueVectors, index, value) -> valueVectors.setSafe(index, (value.getTime() * 1_000)), TimeMicroVector::setNull, Arrays.asList(new Time(-128_000), new Time(104_000), new Time(-42_000))); testSimpleType(new ArrowType.Time(TimeUnit.NANOSECOND, 64), Types.TIME, - (valueVectors, index, value) -> valueVectors.setSafe(index, (int) (value.getTime() * 1_000_000)), + (valueVectors, index, value) -> valueVectors.setSafe(index, (value.getTime() * 1_000_000)), TimeNanoVector::setNull, Arrays.asList(new Time(-128), new Time(104), new Time(-42))); } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowCommentMetadataTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowCommentMetadataTest.java index 07cab0d829fed..4ee65944c3a14 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowCommentMetadataTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowCommentMetadataTest.java @@ -19,9 +19,6 @@ import static org.assertj.core.api.Assertions.assertThat; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.DriverManager; @@ -34,7 +31,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; -import java.util.Objects; import java.util.Set; import org.apache.arrow.memory.RootAllocator; @@ -228,9 +224,4 @@ private String getColumnComment(DatabaseMetaData metaData, String tableName, Str } return null; } - - private String getExpectedSchema(String expectedResource) throws java.io.IOException, java.net.URISyntaxException { - return new String(Files.readAllBytes(Paths.get(Objects.requireNonNull( - JdbcToArrowCommentMetadataTest.class.getResource(expectedResource)).toURI())), StandardCharsets.UTF_8); - } } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigTest.java index 68a681b052cd3..d4fb7c32997a7 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigTest.java @@ -89,8 +89,8 @@ public void testConfig() { JdbcToArrowConfigBuilder builder = new JdbcToArrowConfigBuilder(allocator, calendar); JdbcToArrowConfig config = builder.build(); - assertTrue(allocator == config.getAllocator()); - assertTrue(calendar == config.getCalendar()); + assertEquals(allocator, config.getAllocator()); + assertEquals(calendar, config.getCalendar()); Calendar newCalendar = Calendar.getInstance(); BufferAllocator newAllocator = new RootAllocator(Integer.SIZE); @@ -98,8 +98,8 @@ public void testConfig() { builder.setAllocator(newAllocator).setCalendar(newCalendar); config = builder.build(); - assertTrue(newAllocator == config.getAllocator()); - assertTrue(newCalendar == config.getCalendar()); + assertEquals(newAllocator, config.getAllocator()); + assertEquals(newCalendar, config.getCalendar()); } @Test diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java index 91f2f465dd989..7dd881b3f7cec 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java @@ -24,6 +24,7 @@ import java.math.BigDecimal; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.util.AbstractMap; @@ -399,7 +400,7 @@ public static byte[][] getCharArray(String[] values, String dataType) { byte[][] valueArr = new byte[dataArr.length][]; int i = 0; for (String data : dataArr) { - valueArr[i++] = "null".equals(data.trim()) ? null : data.trim().getBytes(); + valueArr[i++] = "null".equals(data.trim()) ? null : data.trim().getBytes(StandardCharsets.UTF_8); } return valueArr; } @@ -419,11 +420,12 @@ public static byte[][] getBinaryValues(String[] values, String dataType) { byte[][] valueArr = new byte[dataArr.length][]; int i = 0; for (String data : dataArr) { - valueArr[i++] = "null".equals(data.trim()) ? null : data.trim().getBytes(); + valueArr[i++] = "null".equals(data.trim()) ? null : data.trim().getBytes(StandardCharsets.UTF_8); } return valueArr; } + @SuppressWarnings("StringSplitter") public static String[] getValues(String[] values, String dataType) { String value = ""; for (String val : values) { @@ -440,6 +442,7 @@ public static Integer[][] getListValues(String[] values, String dataType) { return getListValues(dataArr); } + @SuppressWarnings("StringSplitter") public static Integer[][] getListValues(String[] dataArr) { Integer[][] valueArr = new Integer[dataArr.length][]; int i = 0; diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ResultSetUtility.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ResultSetUtility.java index ccc7681c5bc8b..b05a59a9a04d8 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ResultSetUtility.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ResultSetUtility.java @@ -66,17 +66,17 @@ public static MockResultSet generateBasicResultSet(int rows) throws SQLException } public static class MockResultSet extends ThrowingResultSet { - private final ArrayList rows; + private final List rows; private int index = 0; private boolean isClosed = false; private ResultSetMetaData metadata; private boolean wasNull; - public MockResultSet(ArrayList rows) throws SQLException { + public MockResultSet(List rows) throws SQLException { this(rows, MockResultSetMetaData.fromRows(rows)); } - public MockResultSet(ArrayList rows, ResultSetMetaData metadata) { + public MockResultSet(List rows, ResultSetMetaData metadata) { this.rows = rows; this.metadata = metadata; this.wasNull = false; @@ -252,8 +252,8 @@ public Builder addDataElement(Object val, int sqlType) { return this.addDataElement(new MockDataElement(val, sqlType)); } - public Builder setMetaData(ResultSetMetaData metaData) { - this.metadata = metaData; + public Builder setMetaData(ResultSetMetaData metadata) { + this.metadata = metadata; return this; } @@ -318,7 +318,7 @@ public String getColumnTypeName(int column) throws SQLException { return columns.get(column - 1).getTypeName(); } - public static MockResultSetMetaData fromRows(ArrayList rows) throws SQLException { + public static MockResultSetMetaData fromRows(List rows) throws SQLException { // Note: This attempts to dynamically construct ResultSetMetaData from the first row in a given result set. // If there are now rows, or the result set contains no columns, this cannot be dynamically generated and // an exception will be thrown. @@ -338,7 +338,6 @@ public static MockResultSetMetaData fromRows(ArrayList rows) throws SQL } public static class MockColumnMetaData { - private int index; private int sqlType; private int precision; private int scale; @@ -385,7 +384,6 @@ private int getDisplaySize() { public static MockColumnMetaData fromDataElement(MockDataElement element, int i) throws SQLException { return MockColumnMetaData.builder() - .index(i) .sqlType(element.getSqlType()) .precision(element.getPrecision()) .scale(element.getScale()) @@ -403,11 +401,6 @@ public static Builder builder() { public static class Builder { private MockColumnMetaData columnMetaData = new MockColumnMetaData(); - public Builder index(int index) { - this.columnMetaData.index = index; - return this; - } - public Builder label(String label) { this.columnMetaData.label = label; return this; @@ -453,9 +446,9 @@ public MockColumnMetaData build() { } public static class MockRow { - private final ArrayList dataElements; + private final List dataElements; - public MockRow(ArrayList elements) { + public MockRow(List elements) { this.dataElements = elements; } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/UnreliableMetaDataTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/UnreliableMetaDataTest.java index 3eb886faabc10..053604073fd66 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/UnreliableMetaDataTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/UnreliableMetaDataTest.java @@ -171,7 +171,6 @@ public void testIncorrectNullability() throws Exception { // ARROW-17005: ResultSetMetaData may indicate a field is non-nullable even when there are nulls ResultSetUtility.MockResultSetMetaData.MockColumnMetaData columnMetaData = ResultSetUtility.MockResultSetMetaData.MockColumnMetaData.builder() - .index(1) .sqlType(Types.INTEGER) .nullable(ResultSetMetaData.columnNoNulls) .build(); @@ -257,7 +256,6 @@ public void testIncorrectNullability() throws Exception { private ResultSet buildIncorrectPrecisionAndScaleMetaDataResultSet() throws SQLException { ResultSetUtility.MockResultSetMetaData.MockColumnMetaData columnMetaData = ResultSetUtility.MockResultSetMetaData.MockColumnMetaData.builder() - .index(1) .sqlType(Types.DECIMAL) .precision(0) .scale(0) @@ -277,7 +275,6 @@ private ResultSet buildIncorrectPrecisionAndScaleMetaDataResultSet() throws SQLE private ResultSet buildVaryingPrecisionAndScaleResultSet() throws SQLException { ResultSetUtility.MockResultSetMetaData.MockColumnMetaData columnMetaData = ResultSetUtility.MockResultSetMetaData.MockColumnMetaData.builder() - .index(1) .sqlType(Types.DECIMAL) .precision(0) .scale(0) diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcAliasToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcAliasToArrowTest.java index d9acfe88f4f8b..d32c2bbab91a8 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcAliasToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcAliasToArrowTest.java @@ -111,8 +111,8 @@ public void testJdbcAliasToArrow() throws Exception { assertEquals(rowCount, vector.getRowCount()); Schema vectorSchema = vector.getSchema(); List vectorFields = vectorSchema.getFields(); - assertEquals(vectorFields.get(0).getName(), COLUMN_A); - assertEquals(vectorFields.get(1).getName(), COLUMN_B); + assertEquals(COLUMN_A, vectorFields.get(0).getName()); + assertEquals(COLUMN_B, vectorFields.get(1).getName()); } } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowArrayTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowArrayTest.java index 377e332b43a13..eabbdc5a25e5d 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowArrayTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowArrayTest.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import java.nio.charset.StandardCharsets; import java.sql.Array; import java.sql.Connection; import java.sql.DriverManager; @@ -284,7 +285,7 @@ private void assertStringVectorEquals(ListVector listVector, int rowCount, Strin assertEquals(1, listVector.isSet(row)); assertEquals(expectedValues[row].length, offset - prevOffset); for (int i = prevOffset; i < offset; ++i) { - assertArrayEquals(expectedValues[row][i - prevOffset].getBytes(), vector.get(i)); + assertArrayEquals(expectedValues[row][i - prevOffset].getBytes(StandardCharsets.UTF_8), vector.get(i)); } prevOffset = offset; diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowCharSetTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowCharSetTest.java index 422b55070aaf9..ab1b4b7fc2fea 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowCharSetTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowCharSetTest.java @@ -76,6 +76,7 @@ public JdbcToArrowCharSetTest(Table table) { * @throws ClassNotFoundException on error */ @Before + @Override public void setUp() throws SQLException, ClassNotFoundException { String url = "jdbc:h2:mem:JdbcToArrowTest?characterEncoding=UTF-8"; String driver = "org.h2.Driver"; @@ -107,6 +108,7 @@ public static Collection getTestData() throws SQLException, ClassNotFo * the multi-byte CJK characters. */ @Test + @Override public void testJdbcToArrowValues() throws SQLException, IOException { testDataSets(sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()), false); @@ -142,6 +144,7 @@ public void testJdbcSchemaMetadata() throws SQLException { * @param isIncludeMapVector is this dataset checks includes map column. * Jdbc type to 'map' mapping declared in configuration only manually */ + @Override public void testDataSets(VectorSchemaRoot root, boolean isIncludeMapVector) { JdbcToArrowTestHelper.assertFieldMetadataIsEmpty(root); diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java index ae4fffd0f94f0..54e7d5ffb27ed 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java @@ -145,6 +145,7 @@ public static Collection getTestData() throws SQLException, ClassNotFo * Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes. */ @Test + @Override public void testJdbcToArrowValues() throws SQLException, IOException { testDataSets(sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()), false); @@ -185,6 +186,7 @@ public void testJdbcSchemaMetadata() throws SQLException { * @param isIncludeMapVector is this dataset checks includes map column. * Jdbc type to 'map' mapping declared in configuration only manually */ + @Override public void testDataSets(VectorSchemaRoot root, boolean isIncludeMapVector) { JdbcToArrowTestHelper.assertFieldMetadataIsEmpty(root); diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowMapDataTypeTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowMapDataTypeTest.java index 43862a93c39c9..a5d1ffa3f64de 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowMapDataTypeTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowMapDataTypeTest.java @@ -45,6 +45,7 @@ public JdbcToArrowMapDataTypeTest() throws IOException { * Test Method to test JdbcToArrow Functionality for Map form Types.OTHER column */ @Test + @Override public void testJdbcToArrowValues() throws SQLException, IOException { Calendar calendar = Calendar.getInstance(); ResultSetMetaData rsmd = getQueryMetaData(table.getQuery()); @@ -68,6 +69,7 @@ public void testJdbcToArrowValues() throws SQLException, IOException { * @param isIncludeMapVector is this dataset checks includes map column. * Jdbc type to 'map' mapping declared in configuration only manually */ + @Override public void testDataSets(VectorSchemaRoot root, boolean isIncludeMapVector) { assertMapVectorValues((MapVector) root.getVector(MAP), table.getRowCount(), getMapValues(table.getValues(), MAP)); diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java index 5731f27c5b345..31d32bd648906 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java @@ -124,6 +124,7 @@ public static Collection getTestData() throws SQLException, ClassNotFo * Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes with null values. */ @Test + @Override public void testJdbcToArrowValues() throws SQLException, IOException { testDataSets(sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()), false); @@ -168,6 +169,7 @@ public void testJdbcSchemaMetadata() throws SQLException { * @param isIncludeMapVector is this dataset checks includes map column. * Jdbc type to 'map' mapping declared in configuration only manually */ + @Override public void testDataSets(VectorSchemaRoot root, boolean isIncludeMapVector) { JdbcToArrowTestHelper.assertFieldMetadataIsEmpty(root); diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowOptionalColumnsTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowOptionalColumnsTest.java index eebcbe64c0e0c..4d0bbfc7a993c 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowOptionalColumnsTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowOptionalColumnsTest.java @@ -70,6 +70,7 @@ public static Collection getTestData() throws SQLException, ClassNotFo * Test Method to test JdbcToArrow Functionality for dealing with nullable and non-nullable columns. */ @Test + @Override public void testJdbcToArrowValues() throws SQLException, IOException { testDataSets(sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE)), false); } @@ -82,6 +83,7 @@ public void testJdbcToArrowValues() throws SQLException, IOException { * @param isIncludeMapVector is this dataset checks includes map column. * Jdbc type to 'map' mapping declared in configuration only manually */ + @Override public void testDataSets(VectorSchemaRoot root, boolean isIncludeMapVector) { JdbcToArrowTestHelper.assertFieldMetadataIsEmpty(root); diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java index 7641fa7f1659c..a925dd7ee32a8 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java @@ -101,6 +101,7 @@ public static Collection getTestData() throws SQLException, ClassNotFo * Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes with only one test data file. */ @Test + @Override public void testJdbcToArrowValues() throws SQLException, IOException { testDataSets(sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()), false); @@ -149,6 +150,7 @@ public void testJdbcSchemaMetadata() throws SQLException { * @param isIncludeMapVector is this dataset checks includes map column. * Jdbc type to 'map' mapping declared in configuration only manually */ + @Override public void testDataSets(VectorSchemaRoot root, boolean isIncludeMapVector) { JdbcToArrowTestHelper.assertFieldMetadataIsEmpty(root); assertBigIntVectorValues((BigIntVector) root.getVector(BIGINT), table.getRowCount(), diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java index 462a75da5143a..fe08db161c8ac 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java @@ -103,6 +103,7 @@ public static Collection getTestData() throws SQLException, ClassNotFo * Time and Timestamp datatype. */ @Test + @Override public void testJdbcToArrowValues() throws SQLException, IOException { testDataSets(sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone()))), false); @@ -139,6 +140,7 @@ public void testJdbcSchemaMetadata() throws SQLException { * @param isIncludeMapVector is this dataset checks includes map column. * Jdbc type to 'map' mapping declared in configuration only manually */ + @Override public void testDataSets(VectorSchemaRoot root, boolean isIncludeMapVector) { JdbcToArrowTestHelper.assertFieldMetadataIsEmpty(root); diff --git a/java/c/src/main/java/org/apache/arrow/c/Format.java b/java/c/src/main/java/org/apache/arrow/c/Format.java index 2875e46f749c4..a5f44859e8327 100644 --- a/java/c/src/main/java/org/apache/arrow/c/Format.java +++ b/java/c/src/main/java/org/apache/arrow/c/Format.java @@ -18,6 +18,7 @@ package org.apache.arrow.c; import java.util.Arrays; +import java.util.Locale; import java.util.stream.Collectors; import org.apache.arrow.util.Preconditions; @@ -127,7 +128,7 @@ static String asString(ArrowType arrowType) { String.format("Int type with bitwidth %d is unsupported", type.getBitWidth())); } if (type.getIsSigned()) { - format = format.toLowerCase(); + format = format.toLowerCase(Locale.ROOT); } return format; } diff --git a/java/c/src/main/java/org/apache/arrow/c/jni/CDataJniException.java b/java/c/src/main/java/org/apache/arrow/c/jni/CDataJniException.java index bebd434f3db3e..df16839c78a16 100644 --- a/java/c/src/main/java/org/apache/arrow/c/jni/CDataJniException.java +++ b/java/c/src/main/java/org/apache/arrow/c/jni/CDataJniException.java @@ -36,10 +36,10 @@ public int getErrno() { } @Override - public String toString() { + public String getMessage() { return "CDataJniException{" + "errno=" + errno + - ", message=" + getMessage() + + ", message=" + super.getMessage() + '}'; } } diff --git a/java/c/src/test/java/org/apache/arrow/c/DictionaryTest.java b/java/c/src/test/java/org/apache/arrow/c/DictionaryTest.java index aa1264e4842eb..48fe2d87f8f3d 100644 --- a/java/c/src/test/java/org/apache/arrow/c/DictionaryTest.java +++ b/java/c/src/test/java/org/apache/arrow/c/DictionaryTest.java @@ -25,6 +25,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.channels.Channels; +import java.nio.charset.StandardCharsets; import java.util.Collections; import org.apache.arrow.c.ArrowArray; @@ -104,9 +105,9 @@ public void testWithDictionary() throws Exception { // create dictionary and provider final VarCharVector dictVector = new VarCharVector("dict", allocator); dictVector.allocateNewSafe(); - dictVector.setSafe(0, "aa".getBytes()); - dictVector.setSafe(1, "bb".getBytes()); - dictVector.setSafe(2, "cc".getBytes()); + dictVector.setSafe(0, "aa".getBytes(StandardCharsets.UTF_8)); + dictVector.setSafe(1, "bb".getBytes(StandardCharsets.UTF_8)); + dictVector.setSafe(2, "cc".getBytes(StandardCharsets.UTF_8)); dictVector.setValueCount(3); Dictionary dictionary = new Dictionary(dictVector, new DictionaryEncoding(0L, false, /* indexType= */null)); @@ -115,10 +116,10 @@ public void testWithDictionary() throws Exception { // create vector and encode it final VarCharVector vector = new VarCharVector("vector", allocator); vector.allocateNewSafe(); - vector.setSafe(0, "bb".getBytes()); - vector.setSafe(1, "bb".getBytes()); - vector.setSafe(2, "cc".getBytes()); - vector.setSafe(3, "aa".getBytes()); + vector.setSafe(0, "bb".getBytes(StandardCharsets.UTF_8)); + vector.setSafe(1, "bb".getBytes(StandardCharsets.UTF_8)); + vector.setSafe(2, "cc".getBytes(StandardCharsets.UTF_8)); + vector.setSafe(3, "aa".getBytes(StandardCharsets.UTF_8)); vector.setValueCount(4); // get the encoded vector @@ -172,11 +173,11 @@ private ArrowStreamReader createMultiBatchReader() throws IOException { // create dictionary and provider DictionaryProvider.MapDictionaryProvider provider = new DictionaryProvider.MapDictionaryProvider(); dictVector.allocateNewSafe(); - dictVector.setSafe(0, "aa".getBytes()); - dictVector.setSafe(1, "bb".getBytes()); - dictVector.setSafe(2, "cc".getBytes()); - dictVector.setSafe(3, "dd".getBytes()); - dictVector.setSafe(4, "ee".getBytes()); + dictVector.setSafe(0, "aa".getBytes(StandardCharsets.UTF_8)); + dictVector.setSafe(1, "bb".getBytes(StandardCharsets.UTF_8)); + dictVector.setSafe(2, "cc".getBytes(StandardCharsets.UTF_8)); + dictVector.setSafe(3, "dd".getBytes(StandardCharsets.UTF_8)); + dictVector.setSafe(4, "ee".getBytes(StandardCharsets.UTF_8)); dictVector.setValueCount(5); Dictionary dictionary = new Dictionary(dictVector, new DictionaryEncoding(0L, false, /* indexType= */null)); provider.put(dictionary); diff --git a/java/c/src/test/java/org/apache/arrow/c/NativeUtilTest.java b/java/c/src/test/java/org/apache/arrow/c/NativeUtilTest.java index f46a0128c8644..9a322b7637922 100644 --- a/java/c/src/test/java/org/apache/arrow/c/NativeUtilTest.java +++ b/java/c/src/test/java/org/apache/arrow/c/NativeUtilTest.java @@ -64,7 +64,7 @@ public void testString() { @Test public void testToJavaArray() { long[] nativeArray = new long[] { 1, 2, 3 }; - try (ArrowBuf buffer = allocator.buffer(Long.BYTES * nativeArray.length, null)) { + try (ArrowBuf buffer = allocator.buffer(Long.BYTES * ((long) nativeArray.length), null)) { for (long value : nativeArray) { buffer.writeLong(value); } diff --git a/java/c/src/test/java/org/apache/arrow/c/RoundtripTest.java b/java/c/src/test/java/org/apache/arrow/c/RoundtripTest.java index 768394ef7ab60..8dcd2ff9a2368 100644 --- a/java/c/src/test/java/org/apache/arrow/c/RoundtripTest.java +++ b/java/c/src/test/java/org/apache/arrow/c/RoundtripTest.java @@ -511,7 +511,7 @@ public void testNullVector() { @Test public void testVarBinaryVector() { try (final VarBinaryVector vector = new VarBinaryVector("v", allocator)) { - setVector(vector, "abc".getBytes(), "def".getBytes(), null); + setVector(vector, "abc".getBytes(StandardCharsets.UTF_8), "def".getBytes(StandardCharsets.UTF_8), null); assertTrue(roundtrip(vector, VarBinaryVector.class)); } } @@ -537,7 +537,7 @@ public void testLargeVarBinaryVector() { String str = "hello world"; try (ArrowBuf buf = allocator.buffer(16)) { - buf.setBytes(0, str.getBytes()); + buf.setBytes(0, str.getBytes(StandardCharsets.UTF_8)); binHolder.start = 0; binHolder.end = str.length(); binHolder.buffer = buf; diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationOutcomeDetails.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationOutcomeDetails.java index 3ceda71cce0fe..fd534b189987c 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationOutcomeDetails.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationOutcomeDetails.java @@ -119,16 +119,14 @@ public boolean isAllocationFailed() { @Override public String toString() { - return new StringBuilder() - .append("allocator[" + accountant.getName() + "]") - .append(" reservation: " + accountant.getInitReservation()) - .append(" limit: " + limit) - .append(" used: " + used) - .append(" requestedSize: " + requestedSize) - .append(" allocatedSize: " + allocatedSize) - .append(" localAllocationStatus: " + (allocationFailed ? "fail" : "success")) - .append("\n") - .toString(); + return "allocator[" + accountant.getName() + "]" + + " reservation: " + accountant.getInitReservation() + + " limit: " + limit + + " used: " + used + + " requestedSize: " + requestedSize + + " allocatedSize: " + allocatedSize + + " localAllocationStatus: " + (allocationFailed ? "fail" : "success") + + "\n"; } } diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationReservation.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationReservation.java index c672dc48d79ca..cc6cbf7e6f2c1 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationReservation.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationReservation.java @@ -84,5 +84,5 @@ public interface AllocationReservation extends AutoCloseable { */ boolean isClosed(); - void close(); + @Override void close(); } diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BaseAllocator.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BaseAllocator.java index 189c800ba0fe5..89b8ffd322a9b 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BaseAllocator.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BaseAllocator.java @@ -532,6 +532,8 @@ public String toVerboseString() { return sb.toString(); } + /* Remove @SuppressWarnings after fixing https://github.com/apache/arrow/issues/41951 */ + @SuppressWarnings("FormatStringAnnotation") private void hist(String noteFormat, Object... args) { if (historicalLog != null) { historicalLog.recordEvent(noteFormat, args); diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferLedger.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferLedger.java index 62d268a1f4493..c610066c982bd 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferLedger.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferLedger.java @@ -58,6 +58,7 @@ boolean isOwningLedger() { return this == allocationManager.getOwningLedger(); } + @Override public BufferAllocator getKey() { return allocator; } @@ -419,7 +420,7 @@ public TransferResult transferOwnership(final ArrowBuf srcBuffer, final BufferAl /** * The outcome of a Transfer. */ - public class TransferResult implements OwnershipTransferResult { + public static class TransferResult implements OwnershipTransferResult { // Whether this transfer fit within the target allocator's capacity. final boolean allocationFit; diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferManager.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferManager.java index 6b622e7192789..e7877d7a3e287 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferManager.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BufferManager.java @@ -49,5 +49,5 @@ public interface BufferManager extends AutoCloseable { */ ArrowBuf getManagedBuffer(long size); - void close(); + @Override void close(); } diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/CheckAllocator.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/CheckAllocator.java index dac4a3fcff59a..bfbf1f212e69a 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/CheckAllocator.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/CheckAllocator.java @@ -60,6 +60,7 @@ static String check() { } } + @SuppressWarnings("URLEqualsHashCode") private static Set scanClasspath() { // LinkedHashSet appropriate here because it preserves insertion order // during iteration diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/ChildAllocator.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/ChildAllocator.java index 67156f89d13aa..f3132cb46a21c 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/ChildAllocator.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/ChildAllocator.java @@ -21,7 +21,7 @@ /** * Child allocator class. Only slightly different from the {@see RootAllocator}, * in that these can't be created directly, but must be obtained from - * {@see BufferAllocator#newChildAllocator(AllocatorOwner, long, long, int)}. + * {@link BufferAllocator#newChildAllocator(String, AllocationListener, long, long)}. * *

Child allocators can only be created by the root, or other children, so * this class is package private.

diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/LowCostIdentityHashMap.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/LowCostIdentityHashMap.java index 740233ef411ff..251028aff0e3c 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/LowCostIdentityHashMap.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/LowCostIdentityHashMap.java @@ -255,7 +255,7 @@ void rehash() { } private void computeMaxSize() { - threshold = (int) ((long) (elementData.length) * LOAD_FACTOR / 10000); + threshold = (int) ((long) elementData.length * LOAD_FACTOR / 10000); } /** @@ -309,7 +309,6 @@ private void computeMaxSize() { elementData[index] = null; } } - return (V) result; } diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/rounding/DefaultRoundingPolicy.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/rounding/DefaultRoundingPolicy.java index 7ba231b0c2095..5939e803fdcd6 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/rounding/DefaultRoundingPolicy.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/rounding/DefaultRoundingPolicy.java @@ -44,20 +44,16 @@ public class DefaultRoundingPolicy implements RoundingPolicy { static { int defaultPageSize = Integer.getInteger("org.apache.memory.allocator.pageSize", 8192); - Throwable pageSizeFallbackCause = null; try { validateAndCalculatePageShifts(defaultPageSize); } catch (Throwable t) { - pageSizeFallbackCause = t; defaultPageSize = 8192; } int defaultMaxOrder = Integer.getInteger("org.apache.memory.allocator.maxOrder", 11); - Throwable maxOrderFallbackCause = null; try { validateAndCalculateChunkSize(defaultPageSize, defaultMaxOrder); } catch (Throwable t) { - maxOrderFallbackCause = t; defaultMaxOrder = 11; } DEFAULT_CHUNK_SIZE = validateAndCalculateChunkSize(defaultPageSize, defaultMaxOrder); diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/ArrowBufPointer.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/ArrowBufPointer.java index b41576847d6b7..5775dd794348b 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/ArrowBufPointer.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/ArrowBufPointer.java @@ -27,7 +27,7 @@ * Pointer to a memory region within an {@link ArrowBuf}. * It will be used as the basis for calculating hash code within a vector, and equality determination. */ -public final class ArrowBufPointer { +public final class ArrowBufPointer implements Comparable { /** * The hash code when the arrow buffer is null. @@ -174,6 +174,7 @@ public int hashCode() { * a positive integer if this pointer is larger; * a negative integer if this pointer is smaller. */ + @Override public int compareTo(ArrowBufPointer that) { if (this.buf == null || that.buf == null) { if (this.buf == null && that.buf == null) { diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/Float16.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/Float16.java index 8040158fd090e..5b80816d48ff7 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/Float16.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/Float16.java @@ -161,7 +161,7 @@ public static float toFloat(short b) { int bits = b & 0xffff; int s = bits & SIGN_MASK; int e = (bits >>> EXPONENT_SHIFT) & SHIFTED_EXPONENT_MASK; - int m = (bits) & SIGNIFICAND_MASK; + int m = bits & SIGNIFICAND_MASK; int outE = 0; int outM = 0; if (e == 0) { // Denormal or 0 @@ -209,7 +209,7 @@ public static short toFloat16(float f) { int bits = Float.floatToRawIntBits(f); int s = (bits >>> FP32_SIGN_SHIFT); int e = (bits >>> FP32_EXPONENT_SHIFT) & FP32_SHIFTED_EXPONENT_MASK; - int m = (bits) & FP32_SIGNIFICAND_MASK; + int m = bits & FP32_SIGNIFICAND_MASK; int outE = 0; int outM = 0; if (e == 0xff) { // Infinite or NaN diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java index 21f063c939ec8..910cc1c21d72d 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java @@ -17,8 +17,9 @@ package org.apache.arrow.memory.util; +import java.util.ArrayDeque; import java.util.Arrays; -import java.util.LinkedList; +import java.util.Deque; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; @@ -30,7 +31,7 @@ */ public class HistoricalLog { - private final LinkedList history = new LinkedList<>(); + private final Deque history = new ArrayDeque<>(); private final String idString; // the formatted id string private final int limit; // the limit on the number of events kept private @Nullable Event firstEvent; // the first stack trace recorded @@ -44,6 +45,8 @@ public class HistoricalLog { * associated with the object instance is best. * @param args for the format string, or nothing if none are required */ + @SuppressWarnings("FormatStringAnnotation") + /* Remove @SuppressWarnings after fixing https://github.com/apache/arrow/issues/41951 */ public HistoricalLog(final String idStringFormat, Object... args) { this(Integer.MAX_VALUE, idStringFormat, args); } @@ -66,7 +69,9 @@ public HistoricalLog(final String idStringFormat, Object... args) { * associated with the object instance is best. * @param args for the format string, or nothing if none are required */ + @SuppressWarnings("AnnotateFormatMethod") public HistoricalLog(final int limit, final String idStringFormat, Object... args) { + // Remove @SuppressWarnings after fixing https://github.com/apache/arrow/issues/41951 this.limit = limit; this.idString = String.format(idStringFormat, args); this.firstEvent = null; @@ -80,7 +85,9 @@ public HistoricalLog(final int limit, final String idStringFormat, Object... arg * @param noteFormat {@link String#format} format string that describes the event * @param args for the format string, or nothing if none are required */ + @SuppressWarnings("AnnotateFormatMethod") public synchronized void recordEvent(final String noteFormat, Object... args) { + // Remove @SuppressWarnings after fixing https://github.com/apache/arrow/issues/41951 final String note = String.format(noteFormat, args); final Event event = new Event(note); if (firstEvent == null) { diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java index 2f74a985a3ff4..727e3531ee83f 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java @@ -58,6 +58,7 @@ public class MemoryUtil { public static final boolean LITTLE_ENDIAN = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN; // Java 1.8, 9, 11, 17, 21 becomes 1, 9, 11, 17, and 21. + @SuppressWarnings("StringSplitter") private static final int majorVersion = Integer.parseInt(System.getProperty("java.specification.version").split("\\D+")[0]); diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/MurmurHasher.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/MurmurHasher.java index 5de98d23bb83b..9c5d0b9086113 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/MurmurHasher.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/MurmurHasher.java @@ -162,7 +162,7 @@ public boolean equals(@Nullable Object o) { if (this == o) { return true; } - if (o == null || getClass() != o.getClass()) { + if (!(o instanceof MurmurHasher)) { return false; } MurmurHasher that = (MurmurHasher) o; diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/SimpleHasher.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/SimpleHasher.java index 3bf3c2a828338..670129d3fb2a2 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/SimpleHasher.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/SimpleHasher.java @@ -53,6 +53,7 @@ protected SimpleHasher() { * @param length length of the memory region. * @return the hash code. */ + @Override public int hashCode(long address, long length) { int hashValue = 0; int index = 0; diff --git a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestArrowBuf.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestArrowBuf.java index b4385b72a38cf..f01d152f84bf3 100644 --- a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestArrowBuf.java +++ b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestArrowBuf.java @@ -141,7 +141,7 @@ public void testSetBytesBigEndian() { } @Test - /** + /* * Test that allocation history is not recorded even though * assertions are enabled in tests (GH-34338). */ diff --git a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java index 535d5c15e8916..1f9e65831b438 100644 --- a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java +++ b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java @@ -374,7 +374,7 @@ public void testCustomizedAllocationManager() { assertEquals(1, arrowBuf1.getInt(0)); try { - final ArrowBuf arrowBuf2 = allocator.buffer(1); + allocator.buffer(1); fail("allocated memory beyond max allowed"); } catch (OutOfMemoryException e) { // expected @@ -1077,7 +1077,7 @@ public void testMemoryLeakWithReservation() throws Exception { "child2", 1024, MAX_ALLOCATION); rootAllocator.verify(); - ArrowBuf buff = childAllocator2.buffer(256); + childAllocator2.buffer(256); Exception exception = assertThrows(IllegalStateException.class, () -> { childAllocator2.close(); diff --git a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestLowCostIdentityHashMap.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestLowCostIdentityHashMap.java index 0cabc4a0571f2..234a6447ddb62 100644 --- a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestLowCostIdentityHashMap.java +++ b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestLowCostIdentityHashMap.java @@ -147,7 +147,7 @@ public void testLargeMap() throws Exception { assertTrue(hashMap.isEmpty()); } - private class StringWithKey implements ValueWithKeyIncluded { + private static class StringWithKey implements ValueWithKeyIncluded { private String myValue; private String myKey; diff --git a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/util/TestArrowBufPointer.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/util/TestArrowBufPointer.java index a1d5624a7e8c0..04e588ed16fc8 100644 --- a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/util/TestArrowBufPointer.java +++ b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/util/TestArrowBufPointer.java @@ -55,16 +55,16 @@ public void testArrowBufPointersEqual() { try (ArrowBuf buf1 = allocator.buffer(BUFFER_LENGTH); ArrowBuf buf2 = allocator.buffer(BUFFER_LENGTH)) { for (int i = 0; i < BUFFER_LENGTH / 4; i++) { - buf1.setInt(i * 4, i * 1234); - buf2.setInt(i * 4, i * 1234); + buf1.setInt(i * 4L, i * 1234); + buf2.setInt(i * 4L, i * 1234); } ArrowBufPointer ptr1 = new ArrowBufPointer(null, 0, 100); ArrowBufPointer ptr2 = new ArrowBufPointer(null, 100, 5032); assertTrue(ptr1.equals(ptr2)); for (int i = 0; i < BUFFER_LENGTH / 4; i++) { - ptr1.set(buf1, i * 4, 4); - ptr2.set(buf2, i * 4, 4); + ptr1.set(buf1, i * 4L, 4); + ptr2.set(buf2, i * 4L, 4); assertTrue(ptr1.equals(ptr2)); } } @@ -76,8 +76,8 @@ public void testArrowBufPointersHashCode() { try (ArrowBuf buf1 = allocator.buffer(vectorLength * 4); ArrowBuf buf2 = allocator.buffer(vectorLength * 4)) { for (int i = 0; i < vectorLength; i++) { - buf1.setInt(i * 4, i); - buf2.setInt(i * 4, i); + buf1.setInt(i * 4L, i); + buf2.setInt(i * 4L, i); } CounterHasher hasher1 = new CounterHasher(); @@ -90,8 +90,8 @@ public void testArrowBufPointersHashCode() { assertEquals(ArrowBufPointer.NULL_HASH_CODE, pointer2.hashCode()); for (int i = 0; i < vectorLength; i++) { - pointer1.set(buf1, i * 4, 4); - pointer2.set(buf2, i * 4, 4); + pointer1.set(buf1, i * 4L, 4); + pointer2.set(buf2, i * 4L, 4); assertEquals(pointer1.hashCode(), pointer2.hashCode()); @@ -188,7 +188,7 @@ public void testArrowBufPointersComparison() { * Hasher with a counter that increments each time a hash code is calculated. * This is to validate that the hash code in {@link ArrowBufPointer} is reused. */ - class CounterHasher implements ArrowBufHasher { + static class CounterHasher implements ArrowBufHasher { protected int counter = 0; @@ -211,7 +211,7 @@ public int hashCode() { @Override public boolean equals(Object o) { - return o != null && this.getClass() == o.getClass(); + return o instanceof CounterHasher; } } } diff --git a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/util/TestByteFunctionHelpers.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/util/TestByteFunctionHelpers.java index 04a715962dfe9..7a44a5f2d72fd 100644 --- a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/util/TestByteFunctionHelpers.java +++ b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/util/TestByteFunctionHelpers.java @@ -19,6 +19,8 @@ import static org.junit.Assert.assertEquals; +import java.nio.charset.StandardCharsets; + import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; @@ -121,9 +123,9 @@ public void testStringCompare() { String rightStr = rightStrings[i]; ArrowBuf left = allocator.buffer(SIZE); - left.setBytes(0, leftStr.getBytes()); + left.setBytes(0, leftStr.getBytes(StandardCharsets.UTF_8)); ArrowBuf right = allocator.buffer(SIZE); - right.setBytes(0, rightStr.getBytes()); + right.setBytes(0, rightStr.getBytes(StandardCharsets.UTF_8)); assertEquals(leftStr.compareTo(rightStr) < 0 ? -1 : 1, ByteFunctionHelpers.compare(left, 0, leftStr.length(), right, 0, rightStr.length())); diff --git a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/util/hash/TestArrowBufHasher.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/util/hash/TestArrowBufHasher.java index 3da0602bdfd9c..cc5ce49e54828 100644 --- a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/util/hash/TestArrowBufHasher.java +++ b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/util/hash/TestArrowBufHasher.java @@ -66,8 +66,8 @@ public void testHasher() { ArrowBuf buf2 = allocator.buffer(BUFFER_LENGTH)) { // prepare data for (int i = 0; i < BUFFER_LENGTH / 4; i++) { - buf1.setFloat(i * 4, i / 10.0f); - buf2.setFloat(i * 4, i / 10.0f); + buf1.setFloat(i * 4L, i / 10.0f); + buf2.setFloat(i * 4L, i / 10.0f); } verifyHashCodesEqual(buf1, 0, 100, buf2, 0, 100); @@ -95,7 +95,7 @@ public void testHasherNegative() { try (ArrowBuf buf = allocator.buffer(BUFFER_LENGTH)) { // prepare data for (int i = 0; i < BUFFER_LENGTH / 4; i++) { - buf.setFloat(i * 4, i / 10.0f); + buf.setFloat(i * 4L, i / 10.0f); } assertThrows(IllegalArgumentException.class, () -> { @@ -120,13 +120,13 @@ public void testHasherLessThanInt() { buf2.writeBytes("bar2".getBytes(StandardCharsets.UTF_8)); for (int i = 1; i <= 4; i ++) { - verifyHashCodeNotEqual(buf1, 0, i, buf2, 0, i); + verifyHashCodeNotEqual(buf1, i, buf2, i); } } } - private void verifyHashCodeNotEqual(ArrowBuf buf1, int offset1, int length1, - ArrowBuf buf2, int offset2, int length2) { + private void verifyHashCodeNotEqual(ArrowBuf buf1, int length1, + ArrowBuf buf2, int length2) { int hashCode1 = hasher.hashCode(buf1, 0, length1); int hashCode2 = hasher.hashCode(buf2, 0, length2); assertNotEquals(hashCode1, hashCode2); diff --git a/java/memory/memory-netty-buffer-patch/src/main/java/io/netty/buffer/NettyArrowBuf.java b/java/memory/memory-netty-buffer-patch/src/main/java/io/netty/buffer/NettyArrowBuf.java index 466444c7d53e8..ae1f30a868406 100644 --- a/java/memory/memory-netty-buffer-patch/src/main/java/io/netty/buffer/NettyArrowBuf.java +++ b/java/memory/memory-netty-buffer-patch/src/main/java/io/netty/buffer/NettyArrowBuf.java @@ -413,7 +413,7 @@ protected int _getUnsignedMedium(int index) { protected int _getUnsignedMediumLE(int index) { this.chk(index, 3); long addr = this.addr(index); - return PlatformDependent.getByte(addr) & 255 | + return (PlatformDependent.getByte(addr) & 255) | (Short.reverseBytes(PlatformDependent.getShort(addr + 1L)) & '\uffff') << 8; } diff --git a/java/memory/memory-netty-buffer-patch/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java b/java/memory/memory-netty-buffer-patch/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java index e51c6c3d48882..ab0cd0c9e6e50 100644 --- a/java/memory/memory-netty-buffer-patch/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java +++ b/java/memory/memory-netty-buffer-patch/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java @@ -36,19 +36,12 @@ public class UnsafeDirectLittleEndian extends WrappedByteBuf { private final AbstractByteBuf wrapped; private final long memoryAddress; - UnsafeDirectLittleEndian(DuplicatedByteBuf buf) { - this(buf, true); - } - - UnsafeDirectLittleEndian(LargeBuffer buf) { - this(buf, true); - } - - UnsafeDirectLittleEndian(PooledUnsafeDirectByteBuf buf) { - this(buf, true); - } - - private UnsafeDirectLittleEndian(AbstractByteBuf buf, boolean fake) { + /** + * Constructs a new instance. + * + * @param buf The buffer to wrap + */ + public UnsafeDirectLittleEndian(AbstractByteBuf buf) { super(buf); this.wrapped = buf; diff --git a/java/memory/memory-netty-buffer-patch/src/test/java/io/netty/buffer/TestUnsafeDirectLittleEndian.java b/java/memory/memory-netty-buffer-patch/src/test/java/io/netty/buffer/TestUnsafeDirectLittleEndian.java index 043c2c1605a63..4717e48f27bef 100644 --- a/java/memory/memory-netty-buffer-patch/src/test/java/io/netty/buffer/TestUnsafeDirectLittleEndian.java +++ b/java/memory/memory-netty-buffer-patch/src/test/java/io/netty/buffer/TestUnsafeDirectLittleEndian.java @@ -23,7 +23,6 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; import org.junit.Test; @@ -34,9 +33,9 @@ import io.netty.buffer.UnsafeDirectLittleEndian; public class TestUnsafeDirectLittleEndian { - private static final boolean LITTLE_ENDIAN = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN; @Test + @SuppressWarnings("CatchAndPrintStackTrace") public void testPrimitiveGetSet() { ByteBuf byteBuf = Unpooled.directBuffer(64); UnsafeDirectLittleEndian unsafeDirect = new UnsafeDirectLittleEndian(new LargeBuffer(byteBuf)); diff --git a/java/memory/memory-netty/src/main/java/org/apache/arrow/memory/netty/NettyAllocationManager.java b/java/memory/memory-netty/src/main/java/org/apache/arrow/memory/netty/NettyAllocationManager.java index 58354d0c2eebd..1e4e06df7e9ac 100644 --- a/java/memory/memory-netty/src/main/java/org/apache/arrow/memory/netty/NettyAllocationManager.java +++ b/java/memory/memory-netty/src/main/java/org/apache/arrow/memory/netty/NettyAllocationManager.java @@ -68,11 +68,9 @@ public ArrowBuf empty() { /** * The cut-off value for switching allocation strategies. */ - private final int allocationCutOffValue; NettyAllocationManager(BufferAllocator accountingAllocator, long requestedSize, int allocationCutOffValue) { super(accountingAllocator); - this.allocationCutOffValue = allocationCutOffValue; if (requestedSize > allocationCutOffValue) { this.memoryChunk = null; @@ -92,7 +90,7 @@ public ArrowBuf empty() { /** * Get the underlying memory chunk managed by this AllocationManager. * @return the underlying memory chunk if the request size is not greater than the - * {@link NettyAllocationManager#allocationCutOffValue}, or null otherwise. + * cutoff value provided in the constructor , or null otherwise. * * @deprecated this method will be removed in a future release. */ diff --git a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/netty/TestEndianness.java b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/netty/TestEndianness.java index a782523cbc6d6..0c99062021f39 100644 --- a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/netty/TestEndianness.java +++ b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/netty/TestEndianness.java @@ -36,15 +36,15 @@ public void testNativeEndian() { final ByteBuf b = NettyArrowBuf.unwrapBuffer(a.buffer(4)); b.setInt(0, 35); if (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN) { - assertEquals(b.getByte(0), 35); - assertEquals(b.getByte(1), 0); - assertEquals(b.getByte(2), 0); - assertEquals(b.getByte(3), 0); + assertEquals(35, b.getByte(0)); + assertEquals(0, b.getByte(1)); + assertEquals(0, b.getByte(2)); + assertEquals(0, b.getByte(3)); } else { - assertEquals(b.getByte(0), 0); - assertEquals(b.getByte(1), 0); - assertEquals(b.getByte(2), 0); - assertEquals(b.getByte(3), 35); + assertEquals(0, b.getByte(0)); + assertEquals(0, b.getByte(1)); + assertEquals(0, b.getByte(2)); + assertEquals(35, b.getByte(3)); } b.release(); a.close(); diff --git a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/netty/TestNettyAllocator.java b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/netty/TestNettyAllocator.java index a6da36bb35aa7..792ae53a9404d 100644 --- a/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/netty/TestNettyAllocator.java +++ b/java/memory/memory-netty/src/test/java/org/apache/arrow/memory/netty/TestNettyAllocator.java @@ -39,6 +39,7 @@ public class TestNettyAllocator { @Test + @SuppressWarnings("SynchronizeOnNonFinalField") public void testMemoryUsage() { ListAppender memoryLogsAppender = new ListAppender<>(); memoryLogsAppender.list = Collections.synchronizedList(memoryLogsAppender.list); diff --git a/java/pom.xml b/java/pom.xml index 9be9d431d4776..a59d29c576398 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -95,6 +95,8 @@ true 9+181-r4173-1 2.24.0 + 2.24.0 + 2.10.0 3.12.1 5.11.0 5.2.0 @@ -995,7 +997,7 @@ UTF-8 -XDcompilePolicy=simple - -Xplugin:ErrorProne -XepExcludedPaths:.*/(target/generated-sources)/.* + -Xplugin:ErrorProne -XepExcludedPaths:.*/(target/generated-source|format/src/main/java/org/apache/arrow/flatbuf)/.* -J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED -J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED -J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED From d02a91b390465dbf530bfba4d100421922b3edda Mon Sep 17 00:00:00 2001 From: mwish Date: Tue, 4 Jun 2024 22:41:32 +0800 Subject: [PATCH 55/93] GH-41608: [C++][Python] Extends the add_key_value to parquet::arrow and PyArrow (#41633) ### Rationale for this change The previous pr ( https://github.com/apache/arrow/pull/34889 ) add a `AddKeyValueMetadata` to FileWriter. And now we should export it to Parquet Arrow and Python API. ### What changes are included in this PR? 1. Add `AddKeyValueMetadata` in parquet::arrow 2. Add `add_key_value_metadata` in pyarrow 3. testing ### Are these changes tested? Yes ### Are there any user-facing changes? New api allowing add key-value metadata to Parquet file * GitHub Issue: #41608 Authored-by: mwish Signed-off-by: mwish --- cpp/src/parquet/CMakeLists.txt | 1 + cpp/src/parquet/arrow/arrow_metadata_test.cc | 97 +++++++++++++++++++ cpp/src/parquet/arrow/writer.cc | 8 ++ cpp/src/parquet/arrow/writer.h | 10 ++ cpp/src/parquet/file_writer.h | 2 +- python/pyarrow/_parquet.pxd | 1 + python/pyarrow/_parquet.pyx | 12 ++- python/pyarrow/parquet/core.py | 13 +++ .../tests/parquet/test_parquet_writer.py | 15 +++ 9 files changed, 157 insertions(+), 2 deletions(-) create mode 100644 cpp/src/parquet/arrow/arrow_metadata_test.cc diff --git a/cpp/src/parquet/CMakeLists.txt b/cpp/src/parquet/CMakeLists.txt index 5ac5085a694c8..dc80f08e72cfe 100644 --- a/cpp/src/parquet/CMakeLists.txt +++ b/cpp/src/parquet/CMakeLists.txt @@ -397,6 +397,7 @@ add_parquet_test(writer-test add_parquet_test(arrow-test SOURCES + arrow/arrow_metadata_test.cc arrow/arrow_reader_writer_test.cc arrow/arrow_schema_test.cc arrow/arrow_statistics_test.cc) diff --git a/cpp/src/parquet/arrow/arrow_metadata_test.cc b/cpp/src/parquet/arrow/arrow_metadata_test.cc new file mode 100644 index 0000000000000..6f512227708b9 --- /dev/null +++ b/cpp/src/parquet/arrow/arrow_metadata_test.cc @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "gtest/gtest.h" + +#include "arrow/table.h" +#include "arrow/testing/gtest_util.h" +#include "arrow/util/key_value_metadata.h" + +#include "parquet/api/writer.h" + +#include "parquet/arrow/reader.h" +#include "parquet/arrow/schema.h" +#include "parquet/arrow/writer.h" +#include "parquet/file_writer.h" +#include "parquet/test_util.h" + +namespace parquet::arrow { + +TEST(Metadata, AppendMetadata) { + // A sample table, type and structure does not matter in this test case + auto schema = ::arrow::schema({::arrow::field("f", ::arrow::utf8())}); + auto table = ::arrow::Table::Make( + schema, {::arrow::ArrayFromJSON(::arrow::utf8(), R"(["a", "b", "c"])")}); + + auto sink = CreateOutputStream(); + ArrowWriterProperties::Builder builder; + builder.store_schema(); + ASSERT_OK_AND_ASSIGN(auto writer, + parquet::arrow::FileWriter::Open( + *schema, ::arrow::default_memory_pool(), sink, + parquet::default_writer_properties(), builder.build())); + + auto kv_meta = std::make_shared(); + kv_meta->Append("test_key_1", "test_value_1"); + // would be overwritten later. + kv_meta->Append("test_key_2", "test_value_2_temp"); + ASSERT_OK(writer->AddKeyValueMetadata(kv_meta)); + + // Key value metadata that will be added to the file. + auto kv_meta_added = std::make_shared<::arrow::KeyValueMetadata>(); + kv_meta_added->Append("test_key_2", "test_value_2"); + kv_meta_added->Append("test_key_3", "test_value_3"); + + ASSERT_OK(writer->AddKeyValueMetadata(kv_meta_added)); + ASSERT_OK(writer->Close()); + + // return error if the file is closed + ASSERT_RAISES(IOError, writer->AddKeyValueMetadata(kv_meta_added)); + + auto verify_key_value_metadata = + [&](const std::shared_ptr& key_value_metadata) { + ASSERT_TRUE(nullptr != key_value_metadata); + + // Verify keys that were added before file writer was closed are present. + for (int i = 1; i <= 3; ++i) { + auto index = std::to_string(i); + PARQUET_ASSIGN_OR_THROW(auto value, + key_value_metadata->Get("test_key_" + index)); + EXPECT_EQ("test_value_" + index, value); + } + EXPECT_TRUE(key_value_metadata->Contains("ARROW:schema")); + }; + // verify the metadata in writer + verify_key_value_metadata(writer->metadata()->key_value_metadata()); + + ASSERT_OK(writer->Close()); + + ASSERT_OK_AND_ASSIGN(auto buffer, sink->Finish()); + // verify the metadata in reader + { + std::unique_ptr reader; + FileReaderBuilder reader_builder; + ASSERT_OK_NO_THROW( + reader_builder.Open(std::make_shared<::arrow::io::BufferReader>(buffer))); + ASSERT_OK( + reader_builder.properties(default_arrow_reader_properties())->Build(&reader)); + + verify_key_value_metadata(reader->parquet_reader()->metadata()->key_value_metadata()); + } +} + +} // namespace parquet::arrow diff --git a/cpp/src/parquet/arrow/writer.cc b/cpp/src/parquet/arrow/writer.cc index bd6f542d11c72..4fd7ef1b47b39 100644 --- a/cpp/src/parquet/arrow/writer.cc +++ b/cpp/src/parquet/arrow/writer.cc @@ -482,6 +482,14 @@ class FileWriterImpl : public FileWriter { return writer_->metadata(); } + /// \brief Append the key-value metadata to the file metadata + ::arrow::Status AddKeyValueMetadata( + const std::shared_ptr& key_value_metadata) + override { + PARQUET_CATCH_NOT_OK(writer_->AddKeyValueMetadata(key_value_metadata)); + return Status::OK(); + } + private: friend class FileWriter; diff --git a/cpp/src/parquet/arrow/writer.h b/cpp/src/parquet/arrow/writer.h index 1decafedc97fd..4a1a033a7b7b8 100644 --- a/cpp/src/parquet/arrow/writer.h +++ b/cpp/src/parquet/arrow/writer.h @@ -143,6 +143,16 @@ class PARQUET_EXPORT FileWriter { virtual ~FileWriter(); virtual MemoryPool* memory_pool() const = 0; + /// \brief Add key-value metadata to the file. + /// \param[in] key_value_metadata the metadata to add. + /// \note This will overwrite any existing metadata with the same key. + /// \return Error if Close() has been called. + /// + /// WARNING: If `store_schema` is enabled, `ARROW:schema` would be stored + /// in the key-value metadata. Overwriting this key would result in + /// `store_schema` being unusable during read. + virtual ::arrow::Status AddKeyValueMetadata( + const std::shared_ptr& key_value_metadata) = 0; /// \brief Return the file metadata, only available after calling Close(). virtual const std::shared_ptr metadata() const = 0; }; diff --git a/cpp/src/parquet/file_writer.h b/cpp/src/parquet/file_writer.h index 31706af86dbde..d5ea1d7c98a0e 100644 --- a/cpp/src/parquet/file_writer.h +++ b/cpp/src/parquet/file_writer.h @@ -202,7 +202,7 @@ class PARQUET_EXPORT ParquetFileWriter { /// \brief Add key-value metadata to the file. /// \param[in] key_value_metadata the metadata to add. - /// \note This will overwrite any existing metadata with the same key. + /// \note This will overwrite any existing metadata with the same key(s). /// \throw ParquetException if Close() has been called. void AddKeyValueMetadata( const std::shared_ptr& key_value_metadata); diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd index ae4094d8b4b5f..1bfa505c54470 100644 --- a/python/pyarrow/_parquet.pxd +++ b/python/pyarrow/_parquet.pxd @@ -554,6 +554,7 @@ cdef extern from "parquet/arrow/writer.h" namespace "parquet::arrow" nogil: CStatus WriteTable(const CTable& table, int64_t chunk_size) CStatus NewRowGroup(int64_t chunk_size) CStatus Close() + CStatus AddKeyValueMetadata(const shared_ptr[const CKeyValueMetadata]& key_value_metadata) const shared_ptr[CFileMetaData] metadata() const diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx index f7724b9b1fdc7..414f0cef4e52b 100644 --- a/python/pyarrow/_parquet.pyx +++ b/python/pyarrow/_parquet.pyx @@ -29,9 +29,10 @@ from pyarrow.includes.libarrow_python cimport * from pyarrow.lib cimport (_Weakrefable, Buffer, Schema, check_status, MemoryPool, maybe_unbox_memory_pool, - Table, NativeFile, + Table, KeyValueMetadata, pyarrow_wrap_chunked_array, pyarrow_wrap_schema, + pyarrow_unwrap_metadata, pyarrow_unwrap_schema, pyarrow_wrap_table, pyarrow_wrap_batch, @@ -2206,6 +2207,15 @@ cdef class ParquetWriter(_Weakrefable): check_status(self.writer.get() .WriteTable(deref(ctable), c_row_group_size)) + def add_key_value_metadata(self, key_value_metadata): + cdef: + shared_ptr[const CKeyValueMetadata] c_metadata + + c_metadata = pyarrow_unwrap_metadata(KeyValueMetadata(key_value_metadata)) + with nogil: + check_status(self.writer.get() + .AddKeyValueMetadata(c_metadata)) + @property def metadata(self): cdef: diff --git a/python/pyarrow/parquet/core.py b/python/pyarrow/parquet/core.py index 81798b1544474..eaff79c8b137c 100644 --- a/python/pyarrow/parquet/core.py +++ b/python/pyarrow/parquet/core.py @@ -1108,6 +1108,19 @@ def close(self): if self.file_handle is not None: self.file_handle.close() + def add_key_value_metadata(self, key_value_metadata): + """ + Add key-value metadata to the file. + This will overwrite any existing metadata with the same key. + + Parameters + ---------- + key_value_metadata : dict + Keys and values must be string-like / coercible to bytes. + """ + assert self.is_open + self.writer.add_key_value_metadata(key_value_metadata) + def _get_pandas_index_columns(keyvalues): return (json.loads(keyvalues[b'pandas'].decode('utf8')) diff --git a/python/pyarrow/tests/parquet/test_parquet_writer.py b/python/pyarrow/tests/parquet/test_parquet_writer.py index f4ee7529ae87d..bc3714a6232b1 100644 --- a/python/pyarrow/tests/parquet/test_parquet_writer.py +++ b/python/pyarrow/tests/parquet/test_parquet_writer.py @@ -346,3 +346,18 @@ def test_parquet_writer_store_schema(tempdir): meta = pq.read_metadata(path2) assert meta.metadata is None + + +def test_parquet_writer_append_key_value_metadata(tempdir): + table = pa.Table.from_arrays([pa.array([], type='int32')], ['f0']) + path = tempdir / 'metadata.parquet' + + with pq.ParquetWriter(path, table.schema) as writer: + writer.write_table(table) + writer.add_key_value_metadata({'key1': '1', 'key2': 'x'}) + writer.add_key_value_metadata({'key2': '2', 'key3': '3'}) + reader = pq.ParquetFile(path) + metadata = reader.metadata.metadata + assert metadata[b'key1'] == b'1' + assert metadata[b'key2'] == b'2' + assert metadata[b'key3'] == b'3' From a44b5372c3933180935e7fbd462fc15a1c298335 Mon Sep 17 00:00:00 2001 From: Haocheng Liu <30446009+HaochengLIU@users.noreply.github.com> Date: Tue, 4 Jun 2024 11:56:13 -0400 Subject: [PATCH 56/93] GH-41493: [C++][S3] Add a new option to check existence before CreateDir (#41822) ### Rationale for this change I have a use case that thousands of jobs are writing hive partitioned parquet files daily to the same bucket via S3FS filesystem. The gist here is a lot of keys are being created at the same time hense jobs hits `AWS Error SLOW_DOWN. during Put Object operation: The object exceeded the rate limit for object mutation operations(create, update, and delete). Please reduce your rate request error.` frequently throughout the day since the code is creating directories pessimistically. ### What changes are included in this PR? Add a new S3Option to check the existence of the directory before creation in `CreateDir`. It's disabled by default. When it's enabled, the CreateDir function will check the existence of the directory first before creation. It ensures that the create operation is only acted when necessary. Though there are more I/O calls, but it avoids hitting the cloud vendor put object limit. ### Are these changes tested? Add test cases when the flag is set to true. Right on top of the mind i donno how to ensure it's working in these tests. But in our production environment, we have very similar code and it worked well. ### Are there any user-facing changes? * GitHub Issue: #41493 Lead-authored-by: Haocheng Liu Co-authored-by: Antoine Pitrou Signed-off-by: Antoine Pitrou --- cpp/src/arrow/filesystem/s3fs.cc | 43 +++++++++++++++++++++++---- cpp/src/arrow/filesystem/s3fs.h | 11 +++++++ cpp/src/arrow/filesystem/s3fs_test.cc | 31 ++++++++++++++++++- 3 files changed, 78 insertions(+), 7 deletions(-) diff --git a/cpp/src/arrow/filesystem/s3fs.cc b/cpp/src/arrow/filesystem/s3fs.cc index 43666f32b3da6..78e02c31a35a3 100644 --- a/cpp/src/arrow/filesystem/s3fs.cc +++ b/cpp/src/arrow/filesystem/s3fs.cc @@ -2860,6 +2860,7 @@ Status S3FileSystem::CreateDir(const std::string& s, bool recursive) { return impl_->CreateBucket(path.bucket); } + FileInfo file_info; // Create object if (recursive) { // Ensure bucket exists @@ -2867,10 +2868,33 @@ Status S3FileSystem::CreateDir(const std::string& s, bool recursive) { if (!bucket_exists) { RETURN_NOT_OK(impl_->CreateBucket(path.bucket)); } + + auto key_i = path.key_parts.begin(); + std::string parent_key{}; + if (options().check_directory_existence_before_creation) { + // Walk up the directory first to find the first existing parent + for (const auto& part : path.key_parts) { + parent_key += part; + parent_key += kSep; + } + for (key_i = path.key_parts.end(); key_i-- != path.key_parts.begin();) { + ARROW_ASSIGN_OR_RAISE(file_info, + this->GetFileInfo(path.bucket + kSep + parent_key)); + if (file_info.type() != FileType::NotFound) { + // Found! + break; + } else { + // remove the kSep and the part + parent_key.pop_back(); + parent_key.erase(parent_key.end() - key_i->size(), parent_key.end()); + } + } + key_i++; // Above for loop moves one extra iterator at the end + } // Ensure that all parents exist, then the directory itself - std::string parent_key; - for (const auto& part : path.key_parts) { - parent_key += part; + // Create all missing directories + for (; key_i < path.key_parts.end(); ++key_i) { + parent_key += *key_i; parent_key += kSep; RETURN_NOT_OK(impl_->CreateEmptyDir(path.bucket, parent_key)); } @@ -2888,11 +2912,18 @@ Status S3FileSystem::CreateDir(const std::string& s, bool recursive) { "': parent directory does not exist"); } } + } - // XXX Should we check that no non-directory entry exists? - // Minio does it for us, not sure about other S3 implementations. - return impl_->CreateEmptyDir(path.bucket, path.key); + // Check if the directory exists already + if (options().check_directory_existence_before_creation) { + ARROW_ASSIGN_OR_RAISE(file_info, this->GetFileInfo(path.full_path)); + if (file_info.type() != FileType::NotFound) { + return Status::OK(); + } } + // XXX Should we check that no non-directory entry exists? + // Minio does it for us, not sure about other S3 implementations. + return impl_->CreateEmptyDir(path.bucket, path.key); } Status S3FileSystem::DeleteDir(const std::string& s) { diff --git a/cpp/src/arrow/filesystem/s3fs.h b/cpp/src/arrow/filesystem/s3fs.h index 82d08bc5ea89a..fbbe9d0b3f42b 100644 --- a/cpp/src/arrow/filesystem/s3fs.h +++ b/cpp/src/arrow/filesystem/s3fs.h @@ -166,6 +166,17 @@ struct ARROW_EXPORT S3Options { /// Whether to allow deletion of buckets bool allow_bucket_deletion = false; + /// Whether to allow pessimistic directory creation in CreateDir function + /// + /// By default, CreateDir function will try to create the directory without checking its + /// existence. It's an optimization to try directory creation and catch the error, + /// rather than issue two dependent I/O calls. + /// Though for key/value storage like Google Cloud Storage, too many creation calls will + /// breach the rate limit for object mutation operations and cause serious consequences. + /// It's also possible you don't have creation access for the parent directory. Set it + /// to be true to address these scenarios. + bool check_directory_existence_before_creation = false; + /// \brief Default metadata for OpenOutputStream. /// /// This will be ignored if non-empty metadata is passed to OpenOutputStream. diff --git a/cpp/src/arrow/filesystem/s3fs_test.cc b/cpp/src/arrow/filesystem/s3fs_test.cc index 88cc96956e34c..7bfa120eda678 100644 --- a/cpp/src/arrow/filesystem/s3fs_test.cc +++ b/cpp/src/arrow/filesystem/s3fs_test.cc @@ -922,9 +922,13 @@ TEST_F(TestS3FS, CreateDir) { // New "directory" AssertFileInfo(fs_.get(), "bucket/newdir", FileType::NotFound); - ASSERT_OK(fs_->CreateDir("bucket/newdir")); + ASSERT_OK(fs_->CreateDir("bucket/newdir", /*recursive=*/false)); AssertFileInfo(fs_.get(), "bucket/newdir", FileType::Directory); + // By default CreateDir uses recursvie mode, make it explictly to be false + ASSERT_RAISES(IOError, + fs_->CreateDir("bucket/newdir/newsub/newsubsub", /*recursive=*/false)); + // New "directory", recursive ASSERT_OK(fs_->CreateDir("bucket/newdir/newsub/newsubsub", /*recursive=*/true)); AssertFileInfo(fs_.get(), "bucket/newdir/newsub", FileType::Directory); @@ -939,6 +943,31 @@ TEST_F(TestS3FS, CreateDir) { // Extraneous slashes ASSERT_RAISES(Invalid, fs_->CreateDir("bucket//somedir")); ASSERT_RAISES(Invalid, fs_->CreateDir("bucket/somedir//newdir")); + + // check existing before creation + options_.check_directory_existence_before_creation = true; + MakeFileSystem(); + // New "directory" again + AssertFileInfo(fs_.get(), "bucket/checknewdir", FileType::NotFound); + ASSERT_OK(fs_->CreateDir("bucket/checknewdir")); + AssertFileInfo(fs_.get(), "bucket/checknewdir", FileType::Directory); + + ASSERT_RAISES(IOError, fs_->CreateDir("bucket/checknewdir/newsub/newsubsub/newsubsub/", + /*recursive=*/false)); + + // New "directory" again, recursive + ASSERT_OK(fs_->CreateDir("bucket/checknewdir/newsub/newsubsub", /*recursive=*/true)); + AssertFileInfo(fs_.get(), "bucket/checknewdir/newsub", FileType::Directory); + AssertFileInfo(fs_.get(), "bucket/checknewdir/newsub/newsubsub", FileType::Directory); + AssertFileInfo(fs_.get(), "bucket/checknewdir/newsub/newsubsub/newsubsub", + FileType::NotFound); + // Try creation with the same name + ASSERT_OK(fs_->CreateDir("bucket/checknewdir/newsub/newsubsub/newsubsub/", + /*recursive=*/true)); + AssertFileInfo(fs_.get(), "bucket/checknewdir/newsub", FileType::Directory); + AssertFileInfo(fs_.get(), "bucket/checknewdir/newsub/newsubsub", FileType::Directory); + AssertFileInfo(fs_.get(), "bucket/checknewdir/newsub/newsubsub/newsubsub", + FileType::Directory); } TEST_F(TestS3FS, DeleteFile) { From ac1eadb5e0291314430583f99727d8c0f6d9b901 Mon Sep 17 00:00:00 2001 From: Tom Scott-Coombes <62209801+tscottcoombes1@users.noreply.github.com> Date: Tue, 4 Jun 2024 23:31:43 +0100 Subject: [PATCH 57/93] GH-40494: [Go] add support for protobuf messages (#40496) ### Rationale for this change Support for protobuf messages ### What changes are included in this PR? Ability to create a schema from a protobuf message Ability to create a record from a protobuf message Some customisations ### Are these changes tested? Yes, couple of unit tests included ### Are there any user-facing changes? No * GitHub Issue: #40494 Lead-authored-by: Tom Scott-Coombes Co-authored-by: Tom Scott-Coombes <62209801+tscottcoombes1@users.noreply.github.com> Co-authored-by: Matt Topol Co-authored-by: tscottcoombes1 <62209801+tscottcoombes1@users.noreply.github.com> Signed-off-by: Matt Topol --- dev/release/rat_exclude_files.txt | 1 + go/arrow/datatype_nested.go | 2 +- go/arrow/util/messages/README.md | 25 + go/arrow/util/messages/types.proto | 56 ++ go/arrow/util/protobuf_reflect.go | 865 +++++++++++++++++++++++++ go/arrow/util/protobuf_reflect_test.go | 311 +++++++++ go/arrow/util/util_message/types.pb.go | 539 +++++++++++++++ go/go.mod | 2 + go/go.sum | 2 + 9 files changed, 1802 insertions(+), 1 deletion(-) create mode 100644 go/arrow/util/messages/README.md create mode 100644 go/arrow/util/messages/types.proto create mode 100644 go/arrow/util/protobuf_reflect.go create mode 100644 go/arrow/util/protobuf_reflect_test.go create mode 100644 go/arrow/util/util_message/types.pb.go diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 0cc1348f50b95..ef325090f2f4b 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -63,6 +63,7 @@ go.work.sum go/go.sum go/arrow/Gopkg.lock go/arrow/flight/gen/flight/*.pb.go +go/arrow/util/util_message/*.pb.go go/arrow/internal/cpu/* go/arrow/type_string.go go/arrow/cdata/test/go.sum diff --git a/go/arrow/datatype_nested.go b/go/arrow/datatype_nested.go index 1e65ccd4594ca..b38983b7f2e5d 100644 --- a/go/arrow/datatype_nested.go +++ b/go/arrow/datatype_nested.go @@ -877,7 +877,7 @@ func DenseUnionFromArrays(children []Array, fields []string, codes []UnionTypeCo } // DenseUnionOf is equivalent to UnionOf(arrow.DenseMode, fields, typeCodes), -// constructing a SparseUnionType from a list of fields and type codes. +// constructing a DenseUnionType from a list of fields and type codes. // // If len(fields) != len(typeCodes) this will panic. They are allowed to be // of length 0. diff --git a/go/arrow/util/messages/README.md b/go/arrow/util/messages/README.md new file mode 100644 index 0000000000000..312484f701a46 --- /dev/null +++ b/go/arrow/util/messages/README.md @@ -0,0 +1,25 @@ + + +How to generate the .pb.go files + +``` +cd go/arrow/util/ +protoc -I ./ --go_out=./messages ./messages/types.proto +``` diff --git a/go/arrow/util/messages/types.proto b/go/arrow/util/messages/types.proto new file mode 100644 index 0000000000000..c085273ca35e0 --- /dev/null +++ b/go/arrow/util/messages/types.proto @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +import "google/protobuf/any.proto"; + +option go_package = "../util_message"; + +message ExampleMessage { + string field1 = 1; +} + +message AllTheTypes { + string str = 1; + int32 int32 = 2; + int64 int64 = 3; + sint32 sint32 = 4; + sint64 sin64 = 5; + uint32 uint32 = 6; + uint64 uint64 = 7; + fixed32 fixed32 = 8; + fixed64 fixed64 = 9; + sfixed32 sfixed32 = 10; + bool bool = 11; + bytes bytes = 12; + double double = 13; + ExampleEnum enum = 14; + ExampleMessage message = 15; + oneof oneof { + string oneofstring = 16; + ExampleMessage oneofmessage = 17; + } + google.protobuf.Any any = 18; + map simple_map = 19; + map complex_map = 20; + repeated string simple_list = 21; + repeated ExampleMessage complex_list = 22; + + enum ExampleEnum { + OPTION_0 = 0; + OPTION_1 = 1; + } +} diff --git a/go/arrow/util/protobuf_reflect.go b/go/arrow/util/protobuf_reflect.go new file mode 100644 index 0000000000000..b4c8d68db8b0d --- /dev/null +++ b/go/arrow/util/protobuf_reflect.go @@ -0,0 +1,865 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "fmt" + "reflect" + + "github.com/apache/arrow/go/v17/arrow" + "github.com/apache/arrow/go/v17/arrow/array" + "github.com/apache/arrow/go/v17/arrow/memory" + "github.com/huandu/xstrings" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/types/known/anypb" +) + +// ProtobufTypeHandler provides options on how protobuf fields should be handled in the conversion to arrow +type ProtobufTypeHandler int + +const ( + // OneOfNull means do not wrap oneOfs in a union, they are treated as separate fields + OneOfNull ProtobufTypeHandler = iota + // OneOfDenseUnion maps the protobuf OneOf to an arrow.DENSE_UNION + OneOfDenseUnion + // EnumNumber uses the Enum numeric value + EnumNumber + // EnumValue uses the Enum string value + EnumValue + // EnumDictionary uses both the numeric and string value and maps to an arrow.Dictionary + EnumDictionary +) + +type schemaOptions struct { + exclusionPolicy func(pfr *ProtobufFieldReflection) bool + fieldNameFormatter func(str string) string + oneOfHandler ProtobufTypeHandler + enumHandler ProtobufTypeHandler +} + +// ProtobufFieldReflection represents the metadata and values of a protobuf field +type ProtobufFieldReflection struct { + parent *ProtobufMessageReflection + descriptor protoreflect.FieldDescriptor + prValue protoreflect.Value + rValue reflect.Value + schemaOptions + arrow.Field +} + +func (pfr *ProtobufFieldReflection) isNull() bool { + for pfr.rValue.Kind() == reflect.Ptr { + if pfr.rValue.IsNil() { + return true + } + pfr.rValue = pfr.rValue.Elem() + } + + if !pfr.rValue.IsValid() || !pfr.prValue.IsValid() { + return true + } + return false +} + +func (pfr *ProtobufFieldReflection) arrowField() arrow.Field { + return arrow.Field{ + Name: pfr.name(), + Type: pfr.getDataType(), + Nullable: true, + } +} + +func (pfr *ProtobufFieldReflection) protoreflectValue() protoreflect.Value { + return pfr.prValue +} + +func (pfr *ProtobufFieldReflection) reflectValue() reflect.Value { + return pfr.rValue +} + +func (pfr *ProtobufFieldReflection) GetDescriptor() protoreflect.FieldDescriptor { + return pfr.descriptor +} + +func (pfr *ProtobufFieldReflection) name() string { + if pfr.isOneOf() && pfr.schemaOptions.oneOfHandler != OneOfNull { + return pfr.fieldNameFormatter(string(pfr.descriptor.ContainingOneof().Name())) + } + return pfr.fieldNameFormatter(string(pfr.descriptor.Name())) +} + +func (pfr *ProtobufFieldReflection) arrowType() arrow.Type { + if pfr.isOneOf() && pfr.schemaOptions.oneOfHandler == OneOfDenseUnion { + return arrow.DENSE_UNION + } + if pfr.isEnum() { + switch pfr.enumHandler { + case EnumNumber: + return arrow.INT32 + case EnumValue: + return arrow.STRING + case EnumDictionary: + return arrow.DICTIONARY + } + } + if pfr.isStruct() { + return arrow.STRUCT + } + if pfr.isMap() { + return arrow.MAP + } + if pfr.isList() { + return arrow.LIST + } + switch pfr.descriptor.Kind() { + case protoreflect.Int32Kind: + return arrow.INT32 + case protoreflect.Int64Kind: + return arrow.INT64 + case protoreflect.Sint32Kind: + return arrow.INT32 + case protoreflect.Sint64Kind: + return arrow.INT64 + case protoreflect.Uint32Kind: + return arrow.UINT32 + case protoreflect.Uint64Kind: + return arrow.UINT64 + case protoreflect.Fixed32Kind: + return arrow.UINT32 + case protoreflect.Fixed64Kind: + return arrow.UINT64 + case protoreflect.Sfixed32Kind: + return arrow.INT32 + case protoreflect.Sfixed64Kind: + return arrow.INT64 + case protoreflect.FloatKind: + return arrow.FLOAT32 + case protoreflect.DoubleKind: + return arrow.FLOAT64 + case protoreflect.StringKind: + return arrow.STRING + case protoreflect.BytesKind: + return arrow.BINARY + case protoreflect.BoolKind: + return arrow.BOOL + } + return arrow.NULL +} + +func (pfr *ProtobufFieldReflection) isOneOf() bool { + return pfr.descriptor.ContainingOneof() != nil +} + +func (pfr *ProtobufFieldReflection) isEnum() bool { + return pfr.descriptor.Kind() == protoreflect.EnumKind +} + +func (pfr *ProtobufFieldReflection) isStruct() bool { + return pfr.descriptor.Kind() == protoreflect.MessageKind && !pfr.descriptor.IsMap() && pfr.rValue.Kind() != reflect.Slice +} + +func (pfr *ProtobufFieldReflection) isMap() bool { + return pfr.descriptor.Kind() == protoreflect.MessageKind && pfr.descriptor.IsMap() +} + +func (pfr *ProtobufFieldReflection) isList() bool { + return pfr.descriptor.IsList() && pfr.rValue.Kind() == reflect.Slice +} + +// ProtobufMessageReflection represents the metadata and values of a protobuf message +type ProtobufMessageReflection struct { + descriptor protoreflect.MessageDescriptor + message protoreflect.Message + rValue reflect.Value + schemaOptions + fields []ProtobufMessageFieldReflection +} + +func (psr ProtobufMessageReflection) unmarshallAny() ProtobufMessageReflection { + if psr.descriptor.FullName() == "google.protobuf.Any" && psr.rValue.IsValid() { + for psr.rValue.Type().Kind() == reflect.Ptr { + psr.rValue = reflect.Indirect(psr.rValue) + } + fieldValueAsAny, _ := psr.rValue.Interface().(anypb.Any) + msg, _ := fieldValueAsAny.UnmarshalNew() + + v := reflect.ValueOf(msg) + for v.Kind() == reflect.Ptr { + v = reflect.Indirect(v) + } + + return ProtobufMessageReflection{ + descriptor: msg.ProtoReflect().Descriptor(), + message: msg.ProtoReflect(), + rValue: v, + schemaOptions: psr.schemaOptions, + } + } else { + return psr + } +} + +func (psr ProtobufMessageReflection) getArrowFields() []arrow.Field { + var fields []arrow.Field + + for pfr := range psr.generateStructFields() { + fields = append(fields, arrow.Field{ + Name: pfr.name(), + Type: pfr.getDataType(), + Nullable: true, + }) + } + + return fields +} + +type protobufListReflection struct { + ProtobufFieldReflection +} + +func (pfr *ProtobufFieldReflection) asList() protobufListReflection { + return protobufListReflection{*pfr} +} + +func (plr protobufListReflection) getDataType() arrow.DataType { + for li := range plr.generateListItems() { + return arrow.ListOf(li.getDataType()) + } + pfr := ProtobufFieldReflection{ + descriptor: plr.descriptor, + schemaOptions: plr.schemaOptions, + } + return arrow.ListOf(pfr.getDataType()) +} + +type protobufUnionReflection struct { + ProtobufFieldReflection +} + +func (pfr *ProtobufFieldReflection) asUnion() protobufUnionReflection { + return protobufUnionReflection{*pfr} +} + +func (pur protobufUnionReflection) isThisOne() bool { + for pur.rValue.Kind() == reflect.Ptr || pur.rValue.Kind() == reflect.Interface { + pur.rValue = pur.rValue.Elem() + } + return pur.rValue.Field(0).String() == pur.prValue.String() +} + +func (pur protobufUnionReflection) whichOne() arrow.UnionTypeCode { + fds := pur.descriptor.ContainingOneof().Fields() + for i := 0; i < fds.Len(); i++ { + pfr := pur.parent.getFieldByName(string(fds.Get(i).Name())) + if pfr.asUnion().isThisOne() { + return pur.getUnionTypeCode(int32(pfr.descriptor.Number())) + } + } + // i.e. all null + return -1 +} + +func (pur protobufUnionReflection) getField() *ProtobufFieldReflection { + fds := pur.descriptor.ContainingOneof().Fields() + for i := 0; i < fds.Len(); i++ { + pfr := pur.parent.getFieldByName(string(fds.Get(i).Name())) + if pfr.asUnion().isThisOne() { + return pfr + } + } + // i.e. all null + return nil +} + +func (pur protobufUnionReflection) getUnionTypeCode(n int32) arrow.UnionTypeCode { + //We use the index of the field number as there is a limit on the arrow.UnionTypeCode (127) + //which a protobuf Number could realistically exceed + fds := pur.descriptor.ContainingOneof().Fields() + for i := 0; i < fds.Len(); i++ { + if n == int32(fds.Get(i).Number()) { + return int8(i) + } + } + return -1 +} + +func (pur protobufUnionReflection) generateUnionFields() chan *ProtobufFieldReflection { + out := make(chan *ProtobufFieldReflection) + go func() { + defer close(out) + fds := pur.descriptor.ContainingOneof().Fields() + for i := 0; i < fds.Len(); i++ { + pfr := pur.parent.getFieldByName(string(fds.Get(i).Name())) + // Do not get stuck in a recursion loop + pfr.oneOfHandler = OneOfNull + if pfr.exclusionPolicy(pfr) { + continue + } + out <- pfr + } + }() + + return out +} + +func (pur protobufUnionReflection) getArrowFields() []arrow.Field { + var fields []arrow.Field + + for pfr := range pur.generateUnionFields() { + fields = append(fields, pfr.arrowField()) + } + + return fields +} + +func (pur protobufUnionReflection) getDataType() arrow.DataType { + fds := pur.getArrowFields() + typeCodes := make([]arrow.UnionTypeCode, len(fds)) + for i := 0; i < len(fds); i++ { + typeCodes[i] = arrow.UnionTypeCode(i) + } + return arrow.DenseUnionOf(fds, typeCodes) +} + +type protobufDictReflection struct { + ProtobufFieldReflection +} + +func (pfr *ProtobufFieldReflection) asDictionary() protobufDictReflection { + return protobufDictReflection{*pfr} +} + +func (pdr protobufDictReflection) getDataType() arrow.DataType { + return &arrow.DictionaryType{ + IndexType: arrow.PrimitiveTypes.Int32, + ValueType: arrow.BinaryTypes.String, + Ordered: false, + } +} + +func (pdr protobufDictReflection) getDictValues(mem memory.Allocator) arrow.Array { + enumValues := pdr.descriptor.Enum().Values() + bldr := array.NewStringBuilder(mem) + for i := 0; i < enumValues.Len(); i++ { + bldr.Append(string(enumValues.Get(i).Name())) + } + return bldr.NewArray() +} + +type protobufMapReflection struct { + ProtobufFieldReflection +} + +func (pfr *ProtobufFieldReflection) asMap() protobufMapReflection { + return protobufMapReflection{*pfr} +} + +func (pmr protobufMapReflection) getDataType() arrow.DataType { + for kvp := range pmr.generateKeyValuePairs() { + return kvp.getDataType() + } + return protobufMapKeyValuePairReflection{ + k: ProtobufFieldReflection{ + parent: pmr.parent, + descriptor: pmr.descriptor.MapKey(), + schemaOptions: pmr.schemaOptions, + }, + v: ProtobufFieldReflection{ + parent: pmr.parent, + descriptor: pmr.descriptor.MapValue(), + schemaOptions: pmr.schemaOptions, + }, + }.getDataType() +} + +type protobufMapKeyValuePairReflection struct { + k ProtobufFieldReflection + v ProtobufFieldReflection +} + +func (pmr protobufMapKeyValuePairReflection) getDataType() arrow.DataType { + return arrow.MapOf(pmr.k.getDataType(), pmr.v.getDataType()) +} + +func (pmr protobufMapReflection) generateKeyValuePairs() chan protobufMapKeyValuePairReflection { + out := make(chan protobufMapKeyValuePairReflection) + + go func() { + defer close(out) + for _, k := range pmr.rValue.MapKeys() { + kvp := protobufMapKeyValuePairReflection{ + k: ProtobufFieldReflection{ + parent: pmr.parent, + descriptor: pmr.descriptor.MapKey(), + prValue: getMapKey(k), + rValue: k, + schemaOptions: pmr.schemaOptions, + }, + v: ProtobufFieldReflection{ + parent: pmr.parent, + descriptor: pmr.descriptor.MapValue(), + prValue: pmr.prValue.Map().Get(protoreflect.MapKey(getMapKey(k))), + rValue: pmr.rValue.MapIndex(k), + schemaOptions: pmr.schemaOptions, + }, + } + out <- kvp + } + }() + + return out +} + +func getMapKey(v reflect.Value) protoreflect.Value { + switch v.Kind() { + case reflect.String: + return protoreflect.ValueOf(v.String()) + case reflect.Int32, reflect.Int64: + return protoreflect.ValueOf(v.Int()) + case reflect.Bool: + return protoreflect.ValueOf(v.Bool()) + case reflect.Uint32, reflect.Uint64: + return protoreflect.ValueOf(v.Uint()) + default: + panic("Unmapped protoreflect map key type") + } +} + +func (psr ProtobufMessageReflection) generateStructFields() chan *ProtobufFieldReflection { + out := make(chan *ProtobufFieldReflection) + + go func() { + defer close(out) + fds := psr.descriptor.Fields() + for i := 0; i < fds.Len(); i++ { + pfr := psr.getFieldByName(string(fds.Get(i).Name())) + if psr.exclusionPolicy(pfr) { + continue + } + if pfr.arrowType() == arrow.DENSE_UNION { + if pfr.descriptor.Number() != pfr.descriptor.ContainingOneof().Fields().Get(0).Number() { + continue + } + } + out <- pfr + } + }() + + return out +} + +func (psr ProtobufMessageReflection) generateFields() chan *ProtobufFieldReflection { + out := make(chan *ProtobufFieldReflection) + + go func() { + defer close(out) + fds := psr.descriptor.Fields() + for i := 0; i < fds.Len(); i++ { + pfr := psr.getFieldByName(string(fds.Get(i).Name())) + if psr.exclusionPolicy(pfr) { + continue + } + if pfr.arrowType() == arrow.DENSE_UNION { + if pfr.descriptor.Number() != pfr.descriptor.ContainingOneof().Fields().Get(0).Number() { + continue + } + } + out <- pfr + } + }() + + return out +} + +func (pfr *ProtobufFieldReflection) asStruct() ProtobufMessageReflection { + psr := ProtobufMessageReflection{ + descriptor: pfr.descriptor.Message(), + rValue: pfr.rValue, + schemaOptions: pfr.schemaOptions, + } + if pfr.prValue.IsValid() { + psr.message = pfr.prValue.Message() + } + psr = psr.unmarshallAny() + return psr +} + +func (psr ProtobufMessageReflection) getDataType() arrow.DataType { + return arrow.StructOf(psr.getArrowFields()...) +} + +func (psr ProtobufMessageReflection) getFieldByName(n string) *ProtobufFieldReflection { + fd := psr.descriptor.Fields().ByTextName(xstrings.ToSnakeCase(n)) + fv := psr.rValue + if fv.IsValid() { + if !fv.IsZero() { + for fv.Kind() == reflect.Ptr || fv.Kind() == reflect.Interface { + fv = fv.Elem() + } + if fd.ContainingOneof() != nil { + n = string(fd.ContainingOneof().Name()) + } + fv = fv.FieldByName(xstrings.ToCamelCase(n)) + for fv.Kind() == reflect.Ptr { + fv = fv.Elem() + } + } + } + pfr := ProtobufFieldReflection{ + parent: &psr, + descriptor: fd, + rValue: fv, + schemaOptions: psr.schemaOptions, + } + if psr.message != nil { + pfr.prValue = psr.message.Get(fd) + } + return &pfr +} + +func (plr protobufListReflection) generateListItems() chan ProtobufFieldReflection { + out := make(chan ProtobufFieldReflection) + + go func() { + defer close(out) + for i := 0; i < plr.prValue.List().Len(); i++ { + out <- ProtobufFieldReflection{ + descriptor: plr.descriptor, + prValue: plr.prValue.List().Get(i), + rValue: plr.rValue.Index(i), + schemaOptions: plr.schemaOptions, + } + } + }() + + return out +} + +func (pfr *ProtobufFieldReflection) getDataType() arrow.DataType { + switch pfr.arrowType() { + case arrow.DENSE_UNION: + return pfr.asUnion().getDataType() + case arrow.DICTIONARY: + return pfr.asDictionary().getDataType() + case arrow.LIST: + return pfr.asList().getDataType() + case arrow.MAP: + return pfr.asMap().getDataType() + case arrow.STRUCT: + return pfr.asStruct().getDataType() + case arrow.INT32: + return arrow.PrimitiveTypes.Int32 + case arrow.INT64: + return arrow.PrimitiveTypes.Int64 + case arrow.UINT32: + return arrow.PrimitiveTypes.Uint32 + case arrow.UINT64: + return arrow.PrimitiveTypes.Uint64 + case arrow.FLOAT32: + return arrow.PrimitiveTypes.Float32 + case arrow.FLOAT64: + return arrow.PrimitiveTypes.Float64 + case arrow.STRING: + return arrow.BinaryTypes.String + case arrow.BINARY: + return arrow.BinaryTypes.Binary + case arrow.BOOL: + return arrow.FixedWidthTypes.Boolean + } + return nil +} + +type protobufReflection interface { + name() string + arrowType() arrow.Type + protoreflectValue() protoreflect.Value + reflectValue() reflect.Value + GetDescriptor() protoreflect.FieldDescriptor + isNull() bool + isEnum() bool + asDictionary() protobufDictReflection + isList() bool + asList() protobufListReflection + isMap() bool + asMap() protobufMapReflection + isStruct() bool + asStruct() ProtobufMessageReflection + isOneOf() bool + asUnion() protobufUnionReflection +} + +// ProtobufMessageFieldReflection links together the message and it's fields +type ProtobufMessageFieldReflection struct { + parent *ProtobufMessageReflection + protobufReflection + arrow.Field +} + +// Schema returns an arrow.Schema representing a protobuf message +func (msg ProtobufMessageReflection) Schema() *arrow.Schema { + var fields []arrow.Field + for _, f := range msg.fields { + fields = append(fields, f.Field) + } + return arrow.NewSchema(fields, nil) +} + +// Record returns an arrow.Record for a protobuf message +func (msg ProtobufMessageReflection) Record(mem memory.Allocator) arrow.Record { + if mem == nil { + mem = memory.NewGoAllocator() + } + + schema := msg.Schema() + + recordBuilder := array.NewRecordBuilder(mem, schema) + + var fieldNames []string + for i, f := range msg.fields { + f.AppendValueOrNull(recordBuilder.Field(i), mem) + fieldNames = append(fieldNames, f.protobufReflection.name()) + } + + var arrays []arrow.Array + for _, bldr := range recordBuilder.Fields() { + a := bldr.NewArray() + arrays = append(arrays, a) + } + + structArray, _ := array.NewStructArray(arrays, fieldNames) + + return array.RecordFromStructArray(structArray, schema) +} + +// NewProtobufMessageReflection initialises a ProtobufMessageReflection +// can be used to convert a protobuf message into an arrow Record +func NewProtobufMessageReflection(msg proto.Message, options ...option) *ProtobufMessageReflection { + v := reflect.ValueOf(msg) + for v.Kind() == reflect.Ptr { + v = v.Elem() + } + includeAll := func(pfr *ProtobufFieldReflection) bool { + return false + } + noFormatting := func(str string) string { + return str + } + psr := &ProtobufMessageReflection{ + descriptor: msg.ProtoReflect().Descriptor(), + message: msg.ProtoReflect(), + rValue: v, + schemaOptions: schemaOptions{ + exclusionPolicy: includeAll, + fieldNameFormatter: noFormatting, + oneOfHandler: OneOfNull, + enumHandler: EnumDictionary, + }, + } + + for _, opt := range options { + opt(psr) + } + + var fields []ProtobufMessageFieldReflection + + for pfr := range psr.generateFields() { + fields = append(fields, ProtobufMessageFieldReflection{ + parent: psr, + protobufReflection: pfr, + Field: pfr.arrowField(), + }) + } + + psr.fields = fields + + return psr +} + +type option func(*ProtobufMessageReflection) + +// WithExclusionPolicy is an option for a ProtobufMessageReflection +// WithExclusionPolicy acts as a deny filter on the fields of a protobuf message +// i.e. prevents them from being included in the schema. +// A use case for this is to exclude fields containing PII. +func WithExclusionPolicy(ex func(pfr *ProtobufFieldReflection) bool) option { + return func(psr *ProtobufMessageReflection) { + psr.exclusionPolicy = ex + } +} + +// WithFieldNameFormatter is an option for a ProtobufMessageReflection +// WithFieldNameFormatter enables customisation of the field names in the arrow schema +// By default, the field names are taken from the protobuf message (.proto file) +func WithFieldNameFormatter(formatter func(str string) string) option { + return func(psr *ProtobufMessageReflection) { + psr.fieldNameFormatter = formatter + } +} + +// WithOneOfHandler is an option for a ProtobufMessageReflection +// WithOneOfHandler enables customisation of the protobuf oneOf type in the arrow schema +// By default, the oneOfs are mapped to separate columns +func WithOneOfHandler(oneOfHandler ProtobufTypeHandler) option { + return func(psr *ProtobufMessageReflection) { + psr.oneOfHandler = oneOfHandler + } +} + +// WithEnumHandler is an option for a ProtobufMessageReflection +// WithEnumHandler enables customisation of the protobuf Enum type in the arrow schema +// By default, the Enums are mapped to arrow.Dictionary +func WithEnumHandler(enumHandler ProtobufTypeHandler) option { + return func(psr *ProtobufMessageReflection) { + psr.enumHandler = enumHandler + } +} + +// AppendValueOrNull add the value of a protobuf field to an arrow array builder +func (f ProtobufMessageFieldReflection) AppendValueOrNull(b array.Builder, mem memory.Allocator) error { + pv := f.protoreflectValue() + fd := f.GetDescriptor() + + if f.isNull() { + b.AppendNull() + return nil + } + + switch b.Type().ID() { + case arrow.STRING: + if f.protobufReflection.isEnum() { + b.(*array.StringBuilder).Append(string(fd.Enum().Values().ByNumber(pv.Enum()).Name())) + } else { + b.(*array.StringBuilder).Append(pv.String()) + } + case arrow.BINARY: + b.(*array.BinaryBuilder).Append(pv.Bytes()) + case arrow.INT32: + if f.protobufReflection.isEnum() { + b.(*array.Int32Builder).Append(int32(f.reflectValue().Int())) + } else { + b.(*array.Int32Builder).Append(int32(pv.Int())) + } + case arrow.INT64: + b.(*array.Int64Builder).Append(pv.Int()) + case arrow.FLOAT64: + b.(*array.Float64Builder).Append(pv.Float()) + case arrow.UINT32: + b.(*array.Uint32Builder).Append(uint32(pv.Uint())) + case arrow.UINT64: + b.(*array.Uint64Builder).Append(pv.Uint()) + case arrow.BOOL: + b.(*array.BooleanBuilder).Append(pv.Bool()) + case arrow.DENSE_UNION: + ub := b.(array.UnionBuilder) + pur := f.asUnion() + if pur.whichOne() == -1 { + ub.AppendNull() + break + } + ub.Append(pur.whichOne()) + cb := ub.Child(int(pur.whichOne())) + err := ProtobufMessageFieldReflection{ + parent: f.parent, + protobufReflection: pur.getField(), + Field: pur.arrowField(), + }.AppendValueOrNull(cb, mem) + if err != nil { + return err + } + case arrow.DICTIONARY: + pdr := f.asDictionary() + db := b.(*array.BinaryDictionaryBuilder) + err := db.InsertStringDictValues(pdr.getDictValues(mem).(*array.String)) + if err != nil { + return err + } + enumNum := int(f.reflectValue().Int()) + enumVal := fd.Enum().Values().ByNumber(protoreflect.EnumNumber(enumNum)).Name() + err = db.AppendValueFromString(string(enumVal)) + if err != nil { + return err + } + case arrow.STRUCT: + sb := b.(*array.StructBuilder) + sb.Append(true) + child := ProtobufMessageFieldReflection{ + parent: f.parent, + } + for i, field := range f.Field.Type.(*arrow.StructType).Fields() { + child.protobufReflection = f.asStruct().getFieldByName(field.Name) + child.Field = field + err := child.AppendValueOrNull(sb.FieldBuilder(i), mem) + if err != nil { + return err + } + } + case arrow.LIST: + lb := b.(*array.ListBuilder) + l := pv.List().Len() + if l == 0 { + lb.AppendEmptyValue() + break + } + lb.ValueBuilder().Reserve(l) + lb.Append(true) + child := ProtobufMessageFieldReflection{ + parent: f.parent, + Field: f.Field.Type.(*arrow.ListType).ElemField(), + } + for li := range f.asList().generateListItems() { + child.protobufReflection = &li + err := child.AppendValueOrNull(lb.ValueBuilder(), mem) + if err != nil { + return err + } + } + case arrow.MAP: + mb := b.(*array.MapBuilder) + l := pv.Map().Len() + if l == 0 { + mb.AppendEmptyValue() + break + } + mb.KeyBuilder().Reserve(l) + mb.ItemBuilder().Reserve(l) + mb.Append(true) + k := ProtobufMessageFieldReflection{ + parent: f.parent, + Field: f.Field.Type.(*arrow.MapType).KeyField(), + } + v := ProtobufMessageFieldReflection{ + parent: f.parent, + Field: f.Field.Type.(*arrow.MapType).ItemField(), + } + for kvp := range f.asMap().generateKeyValuePairs() { + k.protobufReflection = &kvp.k + err := k.AppendValueOrNull(mb.KeyBuilder(), mem) + if err != nil { + return err + } + v.protobufReflection = &kvp.v + err = v.AppendValueOrNull(mb.ItemBuilder(), mem) + if err != nil { + return err + } + } + default: + return fmt.Errorf("not able to appendValueOrNull for type %s", b.Type().ID()) + } + return nil +} diff --git a/go/arrow/util/protobuf_reflect_test.go b/go/arrow/util/protobuf_reflect_test.go new file mode 100644 index 0000000000000..ab3cbdf9a6b13 --- /dev/null +++ b/go/arrow/util/protobuf_reflect_test.go @@ -0,0 +1,311 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "strings" + "testing" + + "github.com/apache/arrow/go/v17/arrow" + "github.com/apache/arrow/go/v17/arrow/array" + "github.com/apache/arrow/go/v17/arrow/memory" + "github.com/apache/arrow/go/v17/arrow/util/util_message" + "github.com/huandu/xstrings" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/anypb" +) + +func SetupTest() util_message.AllTheTypes { + msg := util_message.ExampleMessage{ + Field1: "Example", + } + + anyMsg, _ := anypb.New(&msg) + + return util_message.AllTheTypes{ + Str: "Hello", + Int32: 10, + Int64: 100, + Sint32: -10, + Sin64: -100, + Uint32: 10, + Uint64: 100, + Fixed32: 10, + Fixed64: 1000, + Sfixed32: 10, + Bool: false, + Bytes: []byte("Hello, world!"), + Double: 1.1, + Enum: util_message.AllTheTypes_OPTION_1, + Message: &msg, + Oneof: &util_message.AllTheTypes_Oneofstring{Oneofstring: "World"}, + Any: anyMsg, + //Breaks the test as the Golang maps have a non-deterministic order + //SimpleMap: map[int32]string{99: "Hello", 100: "World", 98: "How", 101: "Are", 1: "You"}, + SimpleMap: map[int32]string{99: "Hello"}, + ComplexMap: map[string]*util_message.ExampleMessage{"complex": &msg}, + SimpleList: []string{"Hello", "World"}, + ComplexList: []*util_message.ExampleMessage{&msg}, + } +} + +func TestGetSchema(t *testing.T) { + msg := SetupTest() + + got := NewProtobufMessageReflection(&msg).Schema().String() + want := `schema: + fields: 22 + - str: type=utf8, nullable + - int32: type=int32, nullable + - int64: type=int64, nullable + - sint32: type=int32, nullable + - sin64: type=int64, nullable + - uint32: type=uint32, nullable + - uint64: type=uint64, nullable + - fixed32: type=uint32, nullable + - fixed64: type=uint64, nullable + - sfixed32: type=int32, nullable + - bool: type=bool, nullable + - bytes: type=binary, nullable + - double: type=float64, nullable + - enum: type=dictionary, nullable + - message: type=struct, nullable + - oneofstring: type=utf8, nullable + - oneofmessage: type=struct, nullable + - any: type=struct, nullable + - simple_map: type=map, nullable + - complex_map: type=map, items_nullable>, nullable + - simple_list: type=list, nullable + - complex_list: type=list, nullable>, nullable` + + require.Equal(t, want, got, "got: %s\nwant: %s", got, want) + + got = NewProtobufMessageReflection(&msg, WithOneOfHandler(OneOfDenseUnion)).Schema().String() + want = `schema: + fields: 21 + - str: type=utf8, nullable + - int32: type=int32, nullable + - int64: type=int64, nullable + - sint32: type=int32, nullable + - sin64: type=int64, nullable + - uint32: type=uint32, nullable + - uint64: type=uint64, nullable + - fixed32: type=uint32, nullable + - fixed64: type=uint64, nullable + - sfixed32: type=int32, nullable + - bool: type=bool, nullable + - bytes: type=binary, nullable + - double: type=float64, nullable + - enum: type=dictionary, nullable + - message: type=struct, nullable + - oneof: type=dense_union, nullable=1>, nullable + - any: type=struct, nullable + - simple_map: type=map, nullable + - complex_map: type=map, items_nullable>, nullable + - simple_list: type=list, nullable + - complex_list: type=list, nullable>, nullable` + + require.Equal(t, want, got, "got: %s\nwant: %s", got, want) + + excludeComplex := func(pfr *ProtobufFieldReflection) bool { + return pfr.isMap() || pfr.isList() || pfr.isStruct() + } + + got = NewProtobufMessageReflection(&msg, WithExclusionPolicy(excludeComplex)).Schema().String() + want = `schema: + fields: 15 + - str: type=utf8, nullable + - int32: type=int32, nullable + - int64: type=int64, nullable + - sint32: type=int32, nullable + - sin64: type=int64, nullable + - uint32: type=uint32, nullable + - uint64: type=uint64, nullable + - fixed32: type=uint32, nullable + - fixed64: type=uint64, nullable + - sfixed32: type=int32, nullable + - bool: type=bool, nullable + - bytes: type=binary, nullable + - double: type=float64, nullable + - enum: type=dictionary, nullable + - oneofstring: type=utf8, nullable` + + require.Equal(t, want, got, "got: %s\nwant: %s", got, want) + + got = NewProtobufMessageReflection( + &msg, + WithExclusionPolicy(excludeComplex), + WithFieldNameFormatter(xstrings.ToCamelCase), + ).Schema().String() + want = `schema: + fields: 15 + - Str: type=utf8, nullable + - Int32: type=int32, nullable + - Int64: type=int64, nullable + - Sint32: type=int32, nullable + - Sin64: type=int64, nullable + - Uint32: type=uint32, nullable + - Uint64: type=uint64, nullable + - Fixed32: type=uint32, nullable + - Fixed64: type=uint64, nullable + - Sfixed32: type=int32, nullable + - Bool: type=bool, nullable + - Bytes: type=binary, nullable + - Double: type=float64, nullable + - Enum: type=dictionary, nullable + - Oneofstring: type=utf8, nullable` + + require.Equal(t, want, got, "got: %s\nwant: %s", got, want) + + onlyEnum := func(pfr *ProtobufFieldReflection) bool { + return !pfr.isEnum() + } + got = NewProtobufMessageReflection( + &msg, + WithExclusionPolicy(onlyEnum), + WithEnumHandler(EnumNumber), + ).Schema().String() + want = `schema: + fields: 1 + - enum: type=int32, nullable` + + require.Equal(t, want, got, "got: %s\nwant: %s", got, want) + + got = NewProtobufMessageReflection( + &msg, + WithExclusionPolicy(onlyEnum), + WithEnumHandler(EnumValue), + ).Schema().String() + want = `schema: + fields: 1 + - enum: type=utf8, nullable` + + require.Equal(t, want, got, "got: %s\nwant: %s", got, want) +} + +func TestRecordFromProtobuf(t *testing.T) { + msg := SetupTest() + + pmr := NewProtobufMessageReflection(&msg, WithOneOfHandler(OneOfDenseUnion)) + schema := pmr.Schema() + got := pmr.Record(nil) + jsonStr := `[ + { + "str":"Hello", + "int32":10, + "int64":100, + "sint32":-10, + "sin64":-100, + "uint32":10, + "uint64":100, + "fixed32":10, + "fixed64":1000, + "sfixed32":10, + "bool":false, + "bytes":"SGVsbG8sIHdvcmxkIQ==", + "double":1.1, + "enum":"OPTION_1", + "message":{"field1":"Example"}, + "oneof": [0, "World"], + "any":{"field1":"Example"}, + "simple_map":[{"key":99,"value":"Hello"}], + "complex_map":[{"key":"complex","value":{"field1":"Example"}}], + "simple_list":["Hello","World"], + "complex_list":[{"field1":"Example"}] + } + ]` + want, _, err := array.RecordFromJSON(memory.NewGoAllocator(), schema, strings.NewReader(jsonStr)) + + require.NoError(t, err) + require.EqualExportedValues(t, got, want, "got: %s\nwant: %s", got, want) + + onlyEnum := func(pfr *ProtobufFieldReflection) bool { return !pfr.isEnum() } + pmr = NewProtobufMessageReflection(&msg, WithExclusionPolicy(onlyEnum), WithEnumHandler(EnumValue)) + got = pmr.Record(nil) + jsonStr = `[ { "enum":"OPTION_1" } ]` + want, _, err = array.RecordFromJSON(memory.NewGoAllocator(), pmr.Schema(), strings.NewReader(jsonStr)) + require.NoError(t, err) + require.True(t, array.RecordEqual(got, want), "got: %s\nwant: %s", got, want) + + pmr = NewProtobufMessageReflection(&msg, WithExclusionPolicy(onlyEnum), WithEnumHandler(EnumNumber)) + got = pmr.Record(nil) + jsonStr = `[ { "enum":"1" } ]` + want, _, err = array.RecordFromJSON(memory.NewGoAllocator(), pmr.Schema(), strings.NewReader(jsonStr)) + require.NoError(t, err) + require.True(t, array.RecordEqual(got, want), "got: %s\nwant: %s", got, want) +} + +func TestNullRecordFromProtobuf(t *testing.T) { + pmr := NewProtobufMessageReflection(&util_message.AllTheTypes{}) + schema := pmr.Schema() + got := pmr.Record(nil) + _, _ = got.MarshalJSON() + jsonStr := `[ + { + "str":"", + "int32":0, + "int64":0, + "sint32":0, + "sin64":0, + "uint32":0, + "uint64":0, + "fixed32":0, + "fixed64":0, + "sfixed32":0, + "bool":false, + "bytes":"", + "double":0, + "enum":"OPTION_0", + "message":null, + "oneofmessage":{"field1":""}, + "oneofstring":"", + "any":null, + "simple_map":[], + "complex_map":[], + "simple_list":[], + "complex_list":[] + } + ]` + + want, _, err := array.RecordFromJSON(memory.NewGoAllocator(), schema, strings.NewReader(jsonStr)) + + require.NoError(t, err) + require.EqualExportedValues(t, got, want, "got: %s\nwant: %s", got, want) +} + +type testProtobufReflection struct { + ProtobufFieldReflection +} + +func (tpr testProtobufReflection) isNull() bool { + return false +} + +func TestAppendValueOrNull(t *testing.T) { + unsupportedField := arrow.Field{Name: "Test", Type: arrow.FixedWidthTypes.Time32s} + schema := arrow.NewSchema([]arrow.Field{unsupportedField}, nil) + mem := memory.NewGoAllocator() + recordBuilder := array.NewRecordBuilder(mem, schema) + pmfr := ProtobufMessageFieldReflection{ + protobufReflection: &testProtobufReflection{}, + Field: arrow.Field{Name: "Test", Type: arrow.FixedWidthTypes.Time32s}, + } + got := pmfr.AppendValueOrNull(recordBuilder.Field(0), mem) + want := "not able to appendValueOrNull for type TIME32" + assert.EqualErrorf(t, got, want, "Error is: %v, want: %v", got, want) +} diff --git a/go/arrow/util/util_message/types.pb.go b/go/arrow/util/util_message/types.pb.go new file mode 100644 index 0000000000000..80e18847c1970 --- /dev/null +++ b/go/arrow/util/util_message/types.pb.go @@ -0,0 +1,539 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.24.4 +// source: messages/types.proto + +package util_message + +import ( + reflect "reflect" + sync "sync" + + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + anypb "google.golang.org/protobuf/types/known/anypb" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type AllTheTypes_ExampleEnum int32 + +const ( + AllTheTypes_OPTION_0 AllTheTypes_ExampleEnum = 0 + AllTheTypes_OPTION_1 AllTheTypes_ExampleEnum = 1 +) + +// Enum value maps for AllTheTypes_ExampleEnum. +var ( + AllTheTypes_ExampleEnum_name = map[int32]string{ + 0: "OPTION_0", + 1: "OPTION_1", + } + AllTheTypes_ExampleEnum_value = map[string]int32{ + "OPTION_0": 0, + "OPTION_1": 1, + } +) + +func (x AllTheTypes_ExampleEnum) Enum() *AllTheTypes_ExampleEnum { + p := new(AllTheTypes_ExampleEnum) + *p = x + return p +} + +func (x AllTheTypes_ExampleEnum) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (AllTheTypes_ExampleEnum) Descriptor() protoreflect.EnumDescriptor { + return file_messages_types_proto_enumTypes[0].Descriptor() +} + +func (AllTheTypes_ExampleEnum) Type() protoreflect.EnumType { + return &file_messages_types_proto_enumTypes[0] +} + +func (x AllTheTypes_ExampleEnum) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use AllTheTypes_ExampleEnum.Descriptor instead. +func (AllTheTypes_ExampleEnum) EnumDescriptor() ([]byte, []int) { + return file_messages_types_proto_rawDescGZIP(), []int{1, 0} +} + +type ExampleMessage struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Field1 string `protobuf:"bytes,1,opt,name=field1,proto3" json:"field1,omitempty"` +} + +func (x *ExampleMessage) Reset() { + *x = ExampleMessage{} + if protoimpl.UnsafeEnabled { + mi := &file_messages_types_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ExampleMessage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExampleMessage) ProtoMessage() {} + +func (x *ExampleMessage) ProtoReflect() protoreflect.Message { + mi := &file_messages_types_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExampleMessage.ProtoReflect.Descriptor instead. +func (*ExampleMessage) Descriptor() ([]byte, []int) { + return file_messages_types_proto_rawDescGZIP(), []int{0} +} + +func (x *ExampleMessage) GetField1() string { + if x != nil { + return x.Field1 + } + return "" +} + +type AllTheTypes struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Str string `protobuf:"bytes,1,opt,name=str,proto3" json:"str,omitempty"` + Int32 int32 `protobuf:"varint,2,opt,name=int32,proto3" json:"int32,omitempty"` + Int64 int64 `protobuf:"varint,3,opt,name=int64,proto3" json:"int64,omitempty"` + Sint32 int32 `protobuf:"zigzag32,4,opt,name=sint32,proto3" json:"sint32,omitempty"` + Sin64 int64 `protobuf:"zigzag64,5,opt,name=sin64,proto3" json:"sin64,omitempty"` + Uint32 uint32 `protobuf:"varint,6,opt,name=uint32,proto3" json:"uint32,omitempty"` + Uint64 uint64 `protobuf:"varint,7,opt,name=uint64,proto3" json:"uint64,omitempty"` + Fixed32 uint32 `protobuf:"fixed32,8,opt,name=fixed32,proto3" json:"fixed32,omitempty"` + Fixed64 uint64 `protobuf:"fixed64,9,opt,name=fixed64,proto3" json:"fixed64,omitempty"` + Sfixed32 int32 `protobuf:"fixed32,10,opt,name=sfixed32,proto3" json:"sfixed32,omitempty"` + Bool bool `protobuf:"varint,11,opt,name=bool,proto3" json:"bool,omitempty"` + Bytes []byte `protobuf:"bytes,12,opt,name=bytes,proto3" json:"bytes,omitempty"` + Double float64 `protobuf:"fixed64,13,opt,name=double,proto3" json:"double,omitempty"` + Enum AllTheTypes_ExampleEnum `protobuf:"varint,14,opt,name=enum,proto3,enum=AllTheTypes_ExampleEnum" json:"enum,omitempty"` + Message *ExampleMessage `protobuf:"bytes,15,opt,name=message,proto3" json:"message,omitempty"` + // Types that are assignable to Oneof: + // + // *AllTheTypes_Oneofstring + // *AllTheTypes_Oneofmessage + Oneof isAllTheTypes_Oneof `protobuf_oneof:"oneof"` + Any *anypb.Any `protobuf:"bytes,18,opt,name=any,proto3" json:"any,omitempty"` + SimpleMap map[int32]string `protobuf:"bytes,19,rep,name=simple_map,json=simpleMap,proto3" json:"simple_map,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + ComplexMap map[string]*ExampleMessage `protobuf:"bytes,20,rep,name=complex_map,json=complexMap,proto3" json:"complex_map,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + SimpleList []string `protobuf:"bytes,21,rep,name=simple_list,json=simpleList,proto3" json:"simple_list,omitempty"` + ComplexList []*ExampleMessage `protobuf:"bytes,22,rep,name=complex_list,json=complexList,proto3" json:"complex_list,omitempty"` +} + +func (x *AllTheTypes) Reset() { + *x = AllTheTypes{} + if protoimpl.UnsafeEnabled { + mi := &file_messages_types_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *AllTheTypes) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AllTheTypes) ProtoMessage() {} + +func (x *AllTheTypes) ProtoReflect() protoreflect.Message { + mi := &file_messages_types_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AllTheTypes.ProtoReflect.Descriptor instead. +func (*AllTheTypes) Descriptor() ([]byte, []int) { + return file_messages_types_proto_rawDescGZIP(), []int{1} +} + +func (x *AllTheTypes) GetStr() string { + if x != nil { + return x.Str + } + return "" +} + +func (x *AllTheTypes) GetInt32() int32 { + if x != nil { + return x.Int32 + } + return 0 +} + +func (x *AllTheTypes) GetInt64() int64 { + if x != nil { + return x.Int64 + } + return 0 +} + +func (x *AllTheTypes) GetSint32() int32 { + if x != nil { + return x.Sint32 + } + return 0 +} + +func (x *AllTheTypes) GetSin64() int64 { + if x != nil { + return x.Sin64 + } + return 0 +} + +func (x *AllTheTypes) GetUint32() uint32 { + if x != nil { + return x.Uint32 + } + return 0 +} + +func (x *AllTheTypes) GetUint64() uint64 { + if x != nil { + return x.Uint64 + } + return 0 +} + +func (x *AllTheTypes) GetFixed32() uint32 { + if x != nil { + return x.Fixed32 + } + return 0 +} + +func (x *AllTheTypes) GetFixed64() uint64 { + if x != nil { + return x.Fixed64 + } + return 0 +} + +func (x *AllTheTypes) GetSfixed32() int32 { + if x != nil { + return x.Sfixed32 + } + return 0 +} + +func (x *AllTheTypes) GetBool() bool { + if x != nil { + return x.Bool + } + return false +} + +func (x *AllTheTypes) GetBytes() []byte { + if x != nil { + return x.Bytes + } + return nil +} + +func (x *AllTheTypes) GetDouble() float64 { + if x != nil { + return x.Double + } + return 0 +} + +func (x *AllTheTypes) GetEnum() AllTheTypes_ExampleEnum { + if x != nil { + return x.Enum + } + return AllTheTypes_OPTION_0 +} + +func (x *AllTheTypes) GetMessage() *ExampleMessage { + if x != nil { + return x.Message + } + return nil +} + +func (m *AllTheTypes) GetOneof() isAllTheTypes_Oneof { + if m != nil { + return m.Oneof + } + return nil +} + +func (x *AllTheTypes) GetOneofstring() string { + if x, ok := x.GetOneof().(*AllTheTypes_Oneofstring); ok { + return x.Oneofstring + } + return "" +} + +func (x *AllTheTypes) GetOneofmessage() *ExampleMessage { + if x, ok := x.GetOneof().(*AllTheTypes_Oneofmessage); ok { + return x.Oneofmessage + } + return nil +} + +func (x *AllTheTypes) GetAny() *anypb.Any { + if x != nil { + return x.Any + } + return nil +} + +func (x *AllTheTypes) GetSimpleMap() map[int32]string { + if x != nil { + return x.SimpleMap + } + return nil +} + +func (x *AllTheTypes) GetComplexMap() map[string]*ExampleMessage { + if x != nil { + return x.ComplexMap + } + return nil +} + +func (x *AllTheTypes) GetSimpleList() []string { + if x != nil { + return x.SimpleList + } + return nil +} + +func (x *AllTheTypes) GetComplexList() []*ExampleMessage { + if x != nil { + return x.ComplexList + } + return nil +} + +type isAllTheTypes_Oneof interface { + isAllTheTypes_Oneof() +} + +type AllTheTypes_Oneofstring struct { + Oneofstring string `protobuf:"bytes,16,opt,name=oneofstring,proto3,oneof"` +} + +type AllTheTypes_Oneofmessage struct { + Oneofmessage *ExampleMessage `protobuf:"bytes,17,opt,name=oneofmessage,proto3,oneof"` +} + +func (*AllTheTypes_Oneofstring) isAllTheTypes_Oneof() {} + +func (*AllTheTypes_Oneofmessage) isAllTheTypes_Oneof() {} + +var File_messages_types_proto protoreflect.FileDescriptor + +var file_messages_types_proto_rawDesc = []byte{ + 0x0a, 0x14, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x19, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x61, 0x6e, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x22, 0x28, 0x0a, 0x0e, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x4d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x31, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x31, 0x22, 0xa9, 0x07, 0x0a, 0x0b, + 0x41, 0x6c, 0x6c, 0x54, 0x68, 0x65, 0x54, 0x79, 0x70, 0x65, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x73, + 0x74, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x74, 0x72, 0x12, 0x14, 0x0a, + 0x05, 0x69, 0x6e, 0x74, 0x33, 0x32, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x69, 0x6e, + 0x74, 0x33, 0x32, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x74, 0x36, 0x34, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x05, 0x69, 0x6e, 0x74, 0x36, 0x34, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x69, 0x6e, + 0x74, 0x33, 0x32, 0x18, 0x04, 0x20, 0x01, 0x28, 0x11, 0x52, 0x06, 0x73, 0x69, 0x6e, 0x74, 0x33, + 0x32, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x69, 0x6e, 0x36, 0x34, 0x18, 0x05, 0x20, 0x01, 0x28, 0x12, + 0x52, 0x05, 0x73, 0x69, 0x6e, 0x36, 0x34, 0x12, 0x16, 0x0a, 0x06, 0x75, 0x69, 0x6e, 0x74, 0x33, + 0x32, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x75, 0x69, 0x6e, 0x74, 0x33, 0x32, 0x12, + 0x16, 0x0a, 0x06, 0x75, 0x69, 0x6e, 0x74, 0x36, 0x34, 0x18, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x06, 0x75, 0x69, 0x6e, 0x74, 0x36, 0x34, 0x12, 0x18, 0x0a, 0x07, 0x66, 0x69, 0x78, 0x65, 0x64, + 0x33, 0x32, 0x18, 0x08, 0x20, 0x01, 0x28, 0x07, 0x52, 0x07, 0x66, 0x69, 0x78, 0x65, 0x64, 0x33, + 0x32, 0x12, 0x18, 0x0a, 0x07, 0x66, 0x69, 0x78, 0x65, 0x64, 0x36, 0x34, 0x18, 0x09, 0x20, 0x01, + 0x28, 0x06, 0x52, 0x07, 0x66, 0x69, 0x78, 0x65, 0x64, 0x36, 0x34, 0x12, 0x1a, 0x0a, 0x08, 0x73, + 0x66, 0x69, 0x78, 0x65, 0x64, 0x33, 0x32, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0f, 0x52, 0x08, 0x73, + 0x66, 0x69, 0x78, 0x65, 0x64, 0x33, 0x32, 0x12, 0x12, 0x0a, 0x04, 0x62, 0x6f, 0x6f, 0x6c, 0x18, + 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x62, 0x6f, 0x6f, 0x6c, 0x12, 0x14, 0x0a, 0x05, 0x62, + 0x79, 0x74, 0x65, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x62, 0x79, 0x74, 0x65, + 0x73, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x18, 0x0d, 0x20, 0x01, 0x28, + 0x01, 0x52, 0x06, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x12, 0x2c, 0x0a, 0x04, 0x65, 0x6e, 0x75, + 0x6d, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x18, 0x2e, 0x41, 0x6c, 0x6c, 0x54, 0x68, 0x65, + 0x54, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x45, 0x6e, 0x75, + 0x6d, 0x52, 0x04, 0x65, 0x6e, 0x75, 0x6d, 0x12, 0x29, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x45, 0x78, 0x61, 0x6d, 0x70, + 0x6c, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x12, 0x22, 0x0a, 0x0b, 0x6f, 0x6e, 0x65, 0x6f, 0x66, 0x73, 0x74, 0x72, 0x69, 0x6e, + 0x67, 0x18, 0x10, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0b, 0x6f, 0x6e, 0x65, 0x6f, 0x66, + 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x12, 0x35, 0x0a, 0x0c, 0x6f, 0x6e, 0x65, 0x6f, 0x66, 0x6d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x45, + 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x48, 0x00, 0x52, + 0x0c, 0x6f, 0x6e, 0x65, 0x6f, 0x66, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x26, 0x0a, + 0x03, 0x61, 0x6e, 0x79, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, + 0x52, 0x03, 0x61, 0x6e, 0x79, 0x12, 0x3a, 0x0a, 0x0a, 0x73, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x5f, + 0x6d, 0x61, 0x70, 0x18, 0x13, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x41, 0x6c, 0x6c, 0x54, + 0x68, 0x65, 0x54, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x4d, 0x61, + 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x09, 0x73, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x4d, 0x61, + 0x70, 0x12, 0x3d, 0x0a, 0x0b, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x5f, 0x6d, 0x61, 0x70, + 0x18, 0x14, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x41, 0x6c, 0x6c, 0x54, 0x68, 0x65, 0x54, + 0x79, 0x70, 0x65, 0x73, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x4d, 0x61, 0x70, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x4d, 0x61, 0x70, + 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, + 0x15, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x4c, 0x69, 0x73, + 0x74, 0x12, 0x32, 0x0a, 0x0c, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x5f, 0x6c, 0x69, 0x73, + 0x74, 0x18, 0x16, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, + 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x52, 0x0b, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, + 0x78, 0x4c, 0x69, 0x73, 0x74, 0x1a, 0x3c, 0x0a, 0x0e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x4d, + 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, + 0x02, 0x38, 0x01, 0x1a, 0x4e, 0x0a, 0x0f, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x4d, 0x61, + 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x25, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, + 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, + 0x02, 0x38, 0x01, 0x22, 0x29, 0x0a, 0x0b, 0x45, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x45, 0x6e, + 0x75, 0x6d, 0x12, 0x0c, 0x0a, 0x08, 0x4f, 0x50, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x30, 0x10, 0x00, + 0x12, 0x0c, 0x0a, 0x08, 0x4f, 0x50, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x31, 0x10, 0x01, 0x42, 0x07, + 0x0a, 0x05, 0x6f, 0x6e, 0x65, 0x6f, 0x66, 0x42, 0x11, 0x5a, 0x0f, 0x2e, 0x2e, 0x2f, 0x75, 0x74, + 0x69, 0x6c, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, +} + +var ( + file_messages_types_proto_rawDescOnce sync.Once + file_messages_types_proto_rawDescData = file_messages_types_proto_rawDesc +) + +func file_messages_types_proto_rawDescGZIP() []byte { + file_messages_types_proto_rawDescOnce.Do(func() { + file_messages_types_proto_rawDescData = protoimpl.X.CompressGZIP(file_messages_types_proto_rawDescData) + }) + return file_messages_types_proto_rawDescData +} + +var file_messages_types_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_messages_types_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_messages_types_proto_goTypes = []interface{}{ + (AllTheTypes_ExampleEnum)(0), // 0: AllTheTypes.ExampleEnum + (*ExampleMessage)(nil), // 1: ExampleMessage + (*AllTheTypes)(nil), // 2: AllTheTypes + nil, // 3: AllTheTypes.SimpleMapEntry + nil, // 4: AllTheTypes.ComplexMapEntry + (*anypb.Any)(nil), // 5: google.protobuf.Any +} +var file_messages_types_proto_depIdxs = []int32{ + 0, // 0: AllTheTypes.enum:type_name -> AllTheTypes.ExampleEnum + 1, // 1: AllTheTypes.message:type_name -> ExampleMessage + 1, // 2: AllTheTypes.oneofmessage:type_name -> ExampleMessage + 5, // 3: AllTheTypes.any:type_name -> google.protobuf.Any + 3, // 4: AllTheTypes.simple_map:type_name -> AllTheTypes.SimpleMapEntry + 4, // 5: AllTheTypes.complex_map:type_name -> AllTheTypes.ComplexMapEntry + 1, // 6: AllTheTypes.complex_list:type_name -> ExampleMessage + 1, // 7: AllTheTypes.ComplexMapEntry.value:type_name -> ExampleMessage + 8, // [8:8] is the sub-list for method output_type + 8, // [8:8] is the sub-list for method input_type + 8, // [8:8] is the sub-list for extension type_name + 8, // [8:8] is the sub-list for extension extendee + 0, // [0:8] is the sub-list for field type_name +} + +func init() { file_messages_types_proto_init() } +func file_messages_types_proto_init() { + if File_messages_types_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_messages_types_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ExampleMessage); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_messages_types_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*AllTheTypes); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_messages_types_proto_msgTypes[1].OneofWrappers = []interface{}{ + (*AllTheTypes_Oneofstring)(nil), + (*AllTheTypes_Oneofmessage)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_messages_types_proto_rawDesc, + NumEnums: 1, + NumMessages: 4, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_messages_types_proto_goTypes, + DependencyIndexes: file_messages_types_proto_depIdxs, + EnumInfos: file_messages_types_proto_enumTypes, + MessageInfos: file_messages_types_proto_msgTypes, + }.Build() + File_messages_types_proto = out.File + file_messages_types_proto_rawDesc = nil + file_messages_types_proto_goTypes = nil + file_messages_types_proto_depIdxs = nil +} diff --git a/go/go.mod b/go/go.mod index b6a3ed207c6ad..b6fccf6735254 100644 --- a/go/go.mod +++ b/go/go.mod @@ -46,7 +46,9 @@ require ( ) require ( + github.com/golang/protobuf v1.5.4 github.com/google/uuid v1.6.0 + github.com/huandu/xstrings v1.4.0 github.com/hamba/avro/v2 v2.22.1 github.com/substrait-io/substrait-go v0.4.2 github.com/tidwall/sjson v1.2.5 diff --git a/go/go.sum b/go/go.sum index 79350f4a1cf27..d963493108d86 100644 --- a/go/go.sum +++ b/go/go.sum @@ -49,6 +49,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= +github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= From e4baf6be2167eb6ccbda90275304336f49998eac Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 5 Jun 2024 01:41:30 +0200 Subject: [PATCH 58/93] GH-38553 : [C++] Replace null_count with MayHaveNulls in ListArrayFromArray and MapArray (#41957) ### Rationale for this change Offsets could have `null_count() == -1` (`kUnknownNullCount`) meaning that offsets might contain nulls that are not accounted for which can produce failures (https://github.com/apache/arrow/issues/38553) when working with `ListArray` or `MapArray`. `null_count()` should be replaced with `MayHaveNulls()`. ### What changes are included in this PR? `null_count` is replaced with `MayHaveNulls` in `ListArrayFromArray`, `MapArray::FromArraysInternal` and `MapArray::ValidateChildData`. Some tests had to be updated. ### Are these changes tested? Yes. ### Are there any user-facing changes? No. * GitHub Issue: #38553 Authored-by: AlenkaF Signed-off-by: Felipe Oliveira Carvalho --- cpp/src/arrow/array/array_list_test.cc | 2 +- cpp/src/arrow/array/array_nested.cc | 8 ++++---- python/pyarrow/tests/test_array.py | 14 +++++++++++++- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/cpp/src/arrow/array/array_list_test.cc b/cpp/src/arrow/array/array_list_test.cc index 55f91dc34167b..063b68706b313 100644 --- a/cpp/src/arrow/array/array_list_test.cc +++ b/cpp/src/arrow/array/array_list_test.cc @@ -1383,7 +1383,7 @@ TEST_F(TestMapArray, FromArrays) { // Null bitmap and offset with offset ASSERT_RAISES(NotImplemented, - MapArray::FromArrays(offsets3->Slice(2), keys, items, pool_, + MapArray::FromArrays(offsets1->Slice(2), keys, items, pool_, offsets3->data()->buffers[0])); } diff --git a/cpp/src/arrow/array/array_nested.cc b/cpp/src/arrow/array/array_nested.cc index bb5c6bf018006..2f6bca3d571ed 100644 --- a/cpp/src/arrow/array/array_nested.cc +++ b/cpp/src/arrow/array/array_nested.cc @@ -115,7 +115,7 @@ Result::ArrayType>> ListArrayFromArray return Status::TypeError("List offsets must be ", OffsetArrowType::type_name()); } - if (null_bitmap != nullptr && offsets.null_count() > 0) { + if (null_bitmap != nullptr && offsets.data()->MayHaveNulls()) { return Status::Invalid( "Ambiguous to specify both validity map and offsets with nulls"); } @@ -827,7 +827,7 @@ Result> MapArray::FromArraysInternal( return Status::Invalid("Map key and item arrays must be equal length"); } - if (null_bitmap != nullptr && offsets->null_count() > 0) { + if (null_bitmap != nullptr && offsets->data()->MayHaveNulls()) { return Status::Invalid( "Ambiguous to specify both validity map and offsets with nulls"); } @@ -893,13 +893,13 @@ Status MapArray::ValidateChildData( if (pair_data->type->id() != Type::STRUCT) { return Status::Invalid("Map array child array should have struct type"); } - if (pair_data->null_count != 0) { + if (pair_data->MayHaveNulls()) { return Status::Invalid("Map array child array should have no nulls"); } if (pair_data->child_data.size() != 2) { return Status::Invalid("Map array child array should have two fields"); } - if (pair_data->child_data[0]->null_count != 0) { + if (pair_data->child_data[0]->MayHaveNulls()) { return Status::Invalid("Map array keys array should have no nulls"); } return Status::OK(); diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py index 49a00517fca9f..88394c77e429d 100644 --- a/python/pyarrow/tests/test_array.py +++ b/python/pyarrow/tests/test_array.py @@ -1013,6 +1013,18 @@ def test_list_array_types_from_arrays_fail(list_array_type, list_type_factory): arr_slice.offsets, arr_slice.values, mask=arr_slice.is_null()) +def test_map_cast(): + # GH-38553 + t = pa.map_(pa.int64(), pa.int64()) + arr = pa.array([{1: 2}], type=t) + result = arr.cast(pa.map_(pa.int32(), pa.int64())) + + t_expected = pa.map_(pa.int32(), pa.int64()) + expected = pa.array([{1: 2}], type=t_expected) + + assert result.equals(expected) + + def test_map_labelled(): # ARROW-13735 t = pa.map_(pa.field("name", "string", nullable=False), "int64") @@ -1105,7 +1117,7 @@ def test_map_from_arrays(): # error if null bitmap passed to sliced offset msg2 = 'Null bitmap with offsets slice not supported.' - offsets = pa.array(offsets, pa.int32()) + offsets = pa.array([0, 2, 2, 6], pa.int32()) with pytest.raises(pa.ArrowNotImplementedError, match=msg2): pa.MapArray.from_arrays(offsets.slice(2), keys, items, pa.map_( keys.type, From fb0773cfcc7d7c3d8b86de022c4d0dc1ae709f91 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jun 2024 10:38:53 +0900 Subject: [PATCH 59/93] MINOR: [JS] Bump eslint-plugin-unicorn from 52.0.0 to 53.0.0 in /js (#41916) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [eslint-plugin-unicorn](https://github.com/sindresorhus/eslint-plugin-unicorn) from 52.0.0 to 53.0.0.
Release notes

Sourced from eslint-plugin-unicorn's releases.

v53.0.0

New rules

Breaking

  • Require Node.js 18.18 (#2250) 598f57b

Meta

Improvements

  • Support ESLint 9 (#2250) 598f57b
  • no-array-method-this-argument: Check Array.fromAsync() (#2330) 99489b9
  • prefer-number-properties: Add checkNaN option (#2315) d30de50
  • template-indent: Support member expression paths in tags and functions (#2346) aabcf1d
  • prefer-number-properties: Don&#39;t require by default for Infinity/-Infinity to be written as Number.POSITIVE_INFINITY/Number.NEGATIVE_INFINITY (#2312) e0dfed2
  • escape-case: Ignore String.raw (#2342) 45bd444
  • no-hex-escape: Ignore String.raw (#2343) cc02a7f
  • prefer-dom-node-dataset: Ignore awaited getAttribute call (#2334) 45f23d5
  • prevent-abbreviations: Support non-ASCII filenames (#2308) 28762c8
  • throw-new-error: Check all call expressions instead of just argument of ThrowStatement (#2332) 1626852

https://github.com/sindresorhus/eslint-plugin-unicorn/compare/v52.0.0...v53.0.0

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=eslint-plugin-unicorn&package-manager=npm_and_yarn&previous-version=52.0.0&new-version=53.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Sutou Kouhei --- js/package.json | 2 +- js/yarn.lock | 78 ++++++++++++++++++++++++++++++++++--------------- 2 files changed, 56 insertions(+), 24 deletions(-) diff --git a/js/package.json b/js/package.json index b55b637e2750d..e7be3c80d82c9 100644 --- a/js/package.json +++ b/js/package.json @@ -83,7 +83,7 @@ "esbuild-plugin-alias": "0.2.1", "eslint": "8.57.0", "eslint-plugin-jest": "28.5.0", - "eslint-plugin-unicorn": "52.0.0", + "eslint-plugin-unicorn": "53.0.0", "esm": "https://github.com/jsg2021/esm/releases/download/v3.x.x-pr883/esm-3.x.x-pr883.tgz", "gulp": "4.0.2", "glob": "10.4.1", diff --git a/js/yarn.lock b/js/yarn.lock index ec311730b8918..3cf3284a9f306 100644 --- a/js/yarn.lock +++ b/js/yarn.lock @@ -174,10 +174,10 @@ resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.24.1.tgz#f99c36d3593db9540705d0739a1f10b5e20c696e" integrity sha512-2ofRCjnnA9y+wk8b9IAREroeUP02KHp431N2mhKniy2yKIDKpbrHv9eXwm8cBeWQYcJmzv5qKCu65P47eCF7CQ== -"@babel/helper-validator-identifier@^7.22.20": - version "7.22.20" - resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz#c4ae002c61d2879e724581d96665583dbc1dc0e0" - integrity sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A== +"@babel/helper-validator-identifier@^7.22.20", "@babel/helper-validator-identifier@^7.24.5": + version "7.24.6" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.24.6.tgz#08bb6612b11bdec78f3feed3db196da682454a5e" + integrity sha512-4yA7s865JHaqUdRbnaxarZREuPTHrjpDT+pXoAZ1yhyo6uFnIEpS8VMu16siFOHDpZNKYv5BObhsB//ycbICyw== "@babel/helper-validator-option@^7.23.5": version "7.23.5" @@ -614,6 +614,21 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" +"@eslint/eslintrc@^3.0.2": + version "3.1.0" + resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-3.1.0.tgz#dbd3482bfd91efa663cbe7aa1f506839868207b6" + integrity sha512-4Bfj15dVJdoy3RfZmmo86RK1Fwzn6SstsvK9JS+BaVKqC6QQQQyXekNaC+g+LKNgkQ+2VhGAzm6hO40AhMR3zQ== + dependencies: + ajv "^6.12.4" + debug "^4.3.2" + espree "^10.0.1" + globals "^14.0.0" + ignore "^5.2.0" + import-fresh "^3.2.1" + js-yaml "^4.1.0" + minimatch "^3.1.2" + strip-json-comments "^3.1.1" + "@eslint/js@8.57.0": version "8.57.0" resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.57.0.tgz#a5417ae8427873f1dd08b70b3574b453e67b5f7f" @@ -1691,7 +1706,7 @@ acorn@^6.4.1: resolved "https://registry.yarnpkg.com/acorn/-/acorn-6.4.2.tgz#35866fd710528e92de10cf06016498e47e39e1e6" integrity sha512-XtGIhXwF8YM8bJhGxG5kXgjkEuNGLTkoYqVE+KMR+aspr4KGYmKYg7yUe3KghyQ9yheNwLnjmzh/7+gfDBmHCQ== -acorn@^8.0.4, acorn@^8.4.1, acorn@^8.7.1, acorn@^8.8.2, acorn@^8.9.0: +acorn@^8.0.4, acorn@^8.11.3, acorn@^8.4.1, acorn@^8.7.1, acorn@^8.8.2, acorn@^8.9.0: version "8.11.3" resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.11.3.tgz#71e0b14e13a4ec160724b38fb7b0f233b1b81d7a" integrity sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg== @@ -2547,10 +2562,10 @@ copy-props@^2.0.1: each-props "^1.3.2" is-plain-object "^5.0.0" -core-js-compat@^3.34.0: - version "3.36.1" - resolved "https://registry.yarnpkg.com/core-js-compat/-/core-js-compat-3.36.1.tgz#1818695d72c99c25d621dca94e6883e190cea3c8" - integrity sha512-Dk997v9ZCt3X/npqzyGdTlq6t7lDBhZwGvV94PKzDArjp7BTRm7WlDAXYd/OWdeFHO8OChQYRJNJvUCqCbrtKA== +core-js-compat@^3.37.0: + version "3.37.1" + resolved "https://registry.yarnpkg.com/core-js-compat/-/core-js-compat-3.37.1.tgz#c844310c7852f4bdf49b8d339730b97e17ff09ee" + integrity sha512-9TNiImhKvQqSUkOvk/mMRZzOANTiEVC7WaBNhHcKM7x+/5E1l5NvsysR19zuDQScE8k+kfQXWRN3AtS/eOSHpg== dependencies: browserslist "^4.23.0" @@ -3022,17 +3037,17 @@ eslint-plugin-jest@28.5.0: dependencies: "@typescript-eslint/utils" "^6.0.0 || ^7.0.0" -eslint-plugin-unicorn@52.0.0: - version "52.0.0" - resolved "https://registry.yarnpkg.com/eslint-plugin-unicorn/-/eslint-plugin-unicorn-52.0.0.tgz#c7a559edd52e3932cf2b3a05c3b0efc604c1eeb8" - integrity sha512-1Yzm7/m+0R4djH0tjDjfVei/ju2w3AzUGjG6q8JnuNIL5xIwsflyCooW5sfBvQp2pMYQFSWWCFONsjCax1EHng== +eslint-plugin-unicorn@53.0.0: + version "53.0.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-unicorn/-/eslint-plugin-unicorn-53.0.0.tgz#df3a5c9ecabeb759e6fd867b2d84198466ac8c4d" + integrity sha512-kuTcNo9IwwUCfyHGwQFOK/HjJAYzbODHN3wP0PgqbW+jbXqpNWxNVpVhj2tO9SixBwuAdmal8rVcWKBxwFnGuw== dependencies: - "@babel/helper-validator-identifier" "^7.22.20" + "@babel/helper-validator-identifier" "^7.24.5" "@eslint-community/eslint-utils" "^4.4.0" - "@eslint/eslintrc" "^2.1.4" + "@eslint/eslintrc" "^3.0.2" ci-info "^4.0.0" clean-regexp "^1.0.0" - core-js-compat "^3.34.0" + core-js-compat "^3.37.0" esquery "^1.5.0" indent-string "^4.0.0" is-builtin-module "^3.2.1" @@ -3041,7 +3056,7 @@ eslint-plugin-unicorn@52.0.0: read-pkg-up "^7.0.1" regexp-tree "^0.1.27" regjsparser "^0.10.0" - semver "^7.5.4" + semver "^7.6.1" strip-indent "^3.0.0" eslint-scope@5.1.1: @@ -3065,6 +3080,11 @@ eslint-visitor-keys@^3.3.0, eslint-visitor-keys@^3.4.1, eslint-visitor-keys@^3.4 resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz#0cd72fe8550e3c2eae156a96a4dddcd1c8ac5800" integrity sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag== +eslint-visitor-keys@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-4.0.0.tgz#e3adc021aa038a2a8e0b2f8b0ce8f66b9483b1fb" + integrity sha512-OtIRv/2GyiF6o/d8K7MYKKbXrOUBIK6SfkIRM4Z0dY3w+LiQ0vy3F57m0Z71bjbyeiWFiHJ8brqnmE6H6/jEuw== + eslint@8.57.0: version "8.57.0" resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.57.0.tgz#c786a6fd0e0b68941aaf624596fb987089195668" @@ -3123,6 +3143,15 @@ esniff@^2.0.1: event-emitter "^0.3.5" type "^2.7.2" +espree@^10.0.1: + version "10.0.1" + resolved "https://registry.yarnpkg.com/espree/-/espree-10.0.1.tgz#600e60404157412751ba4a6f3a2ee1a42433139f" + integrity sha512-MWkrWZbJsL2UwnjxTX3gG8FneachS/Mwg7tdGXce011sJd5b0JG54vat5KHnfSBODZ3Wvzd2WnjxyzsRoVv+ww== + dependencies: + acorn "^8.11.3" + acorn-jsx "^5.3.2" + eslint-visitor-keys "^4.0.0" + espree@^9.6.0, espree@^9.6.1: version "9.6.1" resolved "https://registry.yarnpkg.com/espree/-/espree-9.6.1.tgz#a2a17b8e434690a5432f2f8018ce71d331a48c6f" @@ -3690,6 +3719,11 @@ globals@^13.19.0: dependencies: type-fest "^0.20.2" +globals@^14.0.0: + version "14.0.0" + resolved "https://registry.yarnpkg.com/globals/-/globals-14.0.0.tgz#898d7413c29babcf6bafe56fcadded858ada724e" + integrity sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ== + globby@^11.1.0: version "11.1.0" resolved "https://registry.yarnpkg.com/globby/-/globby-11.1.0.tgz#bd4be98bb042f83d796f7e3811991fbe82a0d34b" @@ -6248,12 +6282,10 @@ semver@^6.3.0, semver@^6.3.1: resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== -semver@^7.3.4, semver@^7.5.3, semver@^7.5.4, semver@^7.6.0: - version "7.6.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.0.tgz#1a46a4db4bffcccd97b743b5005c8325f23d4e2d" - integrity sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg== - dependencies: - lru-cache "^6.0.0" +semver@^7.3.4, semver@^7.5.3, semver@^7.5.4, semver@^7.6.0, semver@^7.6.1: + version "7.6.2" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.2.tgz#1e3b34759f896e8f14d6134732ce798aeb0c6e13" + integrity sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w== serialize-javascript@^6.0.1: version "6.0.2" From ad897bb843b06bb7d54c5913396ac54f736e4e7c Mon Sep 17 00:00:00 2001 From: Anja Kefala Date: Tue, 4 Jun 2024 19:00:05 -0700 Subject: [PATCH 60/93] GH-37929: [Python] begin moving static settings to pyproject.toml (#41041) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change To migrate Arrow to modern Python packaging standards, see [PEP-517](https://peps.python.org/pep-0517/) and [PEP-518](https://peps.python.org/pep-0518/). * GitHub Issue: #37929 This PR focuses on migrating the static settings, the metadata and version, to pyproject.toml. Future PRs will migrate more of the build process to pyproject.toml. Lead-authored-by: anjakefala Co-authored-by: Raúl Cumplido Co-authored-by: Joris Van den Bossche Signed-off-by: Jacob Wujciak-Jens --- .github/workflows/dev.yml | 2 +- ci/conda_env_python.txt | 4 +- dev/release/01-prepare-test.rb | 6 +- dev/release/post-11-bump-versions-test.rb | 6 +- dev/release/utils-prepare.sh | 10 ++- .../python-minimal-build/github.linux.yml | 2 +- .../examples/minimal_build/Dockerfile.ubuntu | 3 +- python/examples/minimal_build/build_conda.sh | 9 +- python/examples/minimal_build/build_venv.sh | 11 +-- python/pyproject.toml | 62 +++++++++++++- python/requirements-build.txt | 4 +- python/setup.py | 85 ------------------- 12 files changed, 89 insertions(+), 115 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 6111d1d2e5fe3..1ea12b0a4d23d 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -120,7 +120,7 @@ jobs: shell: bash run: | gem install test-unit - pip install "cython>=0.29.31" setuptools six pytest jira + pip install "cython>=0.29.31" setuptools six pytest jira setuptools-scm - name: Run Release Test env: ARROW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/ci/conda_env_python.txt b/ci/conda_env_python.txt index 4366e30010389..bf915493de302 100644 --- a/ci/conda_env_python.txt +++ b/ci/conda_env_python.txt @@ -26,5 +26,5 @@ numpy>=1.16.6 pytest pytest-faulthandler s3fs>=2023.10.0 -setuptools -setuptools_scm +setuptools>=64 +setuptools_scm>=8 diff --git a/dev/release/01-prepare-test.rb b/dev/release/01-prepare-test.rb index bf6cfede15c81..fbd0b2996077c 100644 --- a/dev/release/01-prepare-test.rb +++ b/dev/release/01-prepare-test.rb @@ -254,10 +254,10 @@ def test_version_pre_tag ], }, { - path: "python/setup.py", + path: "python/pyproject.toml", hunks: [ - ["-default_version = '#{@snapshot_version}'", - "+default_version = '#{@release_version}'"], + ["-fallback_version = '#{@release_version}a0'", + "+fallback_version = '#{@release_version}'"], ], }, { diff --git a/dev/release/post-11-bump-versions-test.rb b/dev/release/post-11-bump-versions-test.rb index 966c723f70adf..8ad404ef33202 100644 --- a/dev/release/post-11-bump-versions-test.rb +++ b/dev/release/post-11-bump-versions-test.rb @@ -179,10 +179,10 @@ def test_version_post_tag ], }, { - path: "python/setup.py", + path: "python/pyproject.toml", hunks: [ - ["-default_version = '#{@snapshot_version}'", - "+default_version = '#{@next_snapshot_version}'"], + ["-fallback_version = '#{@release_version}a0'", + "+fallback_version = '#{@next_version}a0'"], ], }, { diff --git a/dev/release/utils-prepare.sh b/dev/release/utils-prepare.sh index dfe9b052b09fa..c255e728a335b 100644 --- a/dev/release/utils-prepare.sh +++ b/dev/release/utils-prepare.sh @@ -26,10 +26,12 @@ update_versions() { release) local version=${base_version} local r_version=${base_version} + local python_version=${base_version} ;; snapshot) local version=${next_version}-SNAPSHOT local r_version=${base_version}.9000 + local python_version=${next_version}a0 ;; esac local major_version=${version%%.*} @@ -126,10 +128,10 @@ update_versions() { pushd "${ARROW_DIR}/python" sed -i.bak -E -e \ - "s/^default_version = '.+'/default_version = '${version}'/" \ - setup.py - rm -f setup.py.bak - git add setup.py + "s/^fallback_version = '.+'/fallback_version = '${python_version}'/" \ + pyproject.toml + rm -f pyproject.toml.bak + git add pyproject.toml sed -i.bak -E -e \ "s/^set\(PYARROW_VERSION \".+\"\)/set(PYARROW_VERSION \"${version}\")/" \ CMakeLists.txt diff --git a/dev/tasks/python-minimal-build/github.linux.yml b/dev/tasks/python-minimal-build/github.linux.yml index e776312b93f95..d97968b86b362 100644 --- a/dev/tasks/python-minimal-build/github.linux.yml +++ b/dev/tasks/python-minimal-build/github.linux.yml @@ -26,7 +26,7 @@ jobs: runs-on: ubuntu-latest {{ macros.github_set_env(env) }} steps: - {{ macros.github_checkout_arrow(submodules=false)|indent }} + {{ macros.github_checkout_arrow(fetch_depth=0, submodules=false)|indent }} - name: Run minimal build example run: | diff --git a/python/examples/minimal_build/Dockerfile.ubuntu b/python/examples/minimal_build/Dockerfile.ubuntu index ebea4b045e592..07cd69c082461 100644 --- a/python/examples/minimal_build/Dockerfile.ubuntu +++ b/python/examples/minimal_build/Dockerfile.ubuntu @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM ubuntu:focal +FROM ubuntu:jammy ENV DEBIAN_FRONTEND=noninteractive @@ -32,6 +32,7 @@ RUN apt-get update -y -q && \ python3-dev \ python3-pip \ python3-venv \ + tzdata \ && \ apt-get clean && rm -rf /var/lib/apt/lists* diff --git a/python/examples/minimal_build/build_conda.sh b/python/examples/minimal_build/build_conda.sh index 72c3a5f9ea2cd..e90c800ae2eb1 100755 --- a/python/examples/minimal_build/build_conda.sh +++ b/python/examples/minimal_build/build_conda.sh @@ -97,9 +97,8 @@ export CMAKE_PREFIX_PATH=${ARROW_HOME}${CMAKE_PREFIX_PATH:+:${CMAKE_PREFIX_PATH} export PYARROW_BUILD_TYPE=Debug export PYARROW_CMAKE_GENERATOR=Ninja -# You can run either "develop" or "build_ext --inplace". Your pick - -# python setup.py build_ext --inplace -python setup.py develop +# Use the same command that we use on python_build.sh +python -m pip install --no-deps --no-build-isolation -vv . +popd -py.test pyarrow +pytest -vv -r s ${PYTEST_ARGS} --pyargs pyarrow diff --git a/python/examples/minimal_build/build_venv.sh b/python/examples/minimal_build/build_venv.sh index 3bd641d0e72c9..f462c4e9b9d0a 100755 --- a/python/examples/minimal_build/build_venv.sh +++ b/python/examples/minimal_build/build_venv.sh @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -set -e +set -ex #---------------------------------------------------------------------- # Change this to whatever makes sense for your system @@ -35,6 +35,7 @@ source $WORKDIR/venv/bin/activate git config --global --add safe.directory $ARROW_ROOT pip install -r $ARROW_ROOT/python/requirements-build.txt +pip install wheel #---------------------------------------------------------------------- # Build C++ library @@ -68,11 +69,11 @@ export CMAKE_PREFIX_PATH=${ARROW_HOME}${CMAKE_PREFIX_PATH:+:${CMAKE_PREFIX_PATH} export PYARROW_BUILD_TYPE=Debug export PYARROW_CMAKE_GENERATOR=Ninja -# You can run either "develop" or "build_ext --inplace". Your pick +# Use the same command that we use on python_build.sh +python -m pip install --no-deps --no-build-isolation -vv . -# python setup.py build_ext --inplace -python setup.py develop +popd pip install -r $ARROW_ROOT/python/requirements-test.txt -py.test pyarrow +pytest -vv -r s ${PYTEST_ARGS} --pyargs pyarrow diff --git a/python/pyproject.toml b/python/pyproject.toml index 1588e690a7247..f72c3a91eb436 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -24,7 +24,63 @@ requires = [ # continue using oldest-support-numpy. "oldest-supported-numpy>=0.14; python_version<'3.9'", "numpy>=1.25; python_version>='3.9'", - "setuptools_scm", - "setuptools >= 40.1.0", - "wheel" + # configuring setuptools_scm in pyproject.toml requires + # versions released after 2022 + "setuptools_scm[toml]>=8", + "setuptools>=64", ] +build-backend = "setuptools.build_meta" + +[project] +name = "pyarrow" +dynamic = ["version"] +requires-python = ">=3.8" +dependencies = [ + "numpy >= 1.16.6" +] +description = "Python library for Apache Arrow" +readme = {file = "README.md", content-type = "text/markdown"} +license = {text = "Apache Software License"} +classifiers = [ + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Langauge :: Python :: 3.12', +] +maintainers = [ + {name = "Apache Arrow Developers", email = "dev@arrow.apache.org"} +] + +[project.urls] +Homepage = "https://arrow.apache.org/" +Documentation = "https://arrow.apache.org/docs/python" +Repository = "https://github.com/apache/arrow" +Issues = "https://github.com/apache/arrow/issues" + +[project.optional-dependencies] +test = [ + 'pytest', + 'hypothesis', + 'cffi', + 'pytz', + 'pandas' +] + +[tool.setuptools] +zip-safe=false +include-package-data=true + +[tool.setuptools.packages.find] +where = ["."] + +[tool.setuptools.package-data] +pyarrow = ["*.pxd", "*.pyx", "includes/*.pxd"] + +[tool.setuptools_scm] +root = '..' +version_file = 'pyarrow/_generated_version.py' +version_scheme = 'guess-next-dev' +git_describe_command = 'git describe --dirty --tags --long --match "apache-arrow-[0-9]*.*"' +fallback_version = '17.0.0a0' diff --git a/python/requirements-build.txt b/python/requirements-build.txt index 87dcc148ad161..c150c842a0cc6 100644 --- a/python/requirements-build.txt +++ b/python/requirements-build.txt @@ -1,5 +1,5 @@ cython>=0.29.31 oldest-supported-numpy>=0.14; python_version<'3.9' numpy>=1.25; python_version>='3.9' -setuptools_scm -setuptools>=38.6.0 +setuptools_scm>=8 +setuptools>=64 diff --git a/python/setup.py b/python/setup.py index ed2b7961e5fbb..b738b2f77290e 100755 --- a/python/setup.py +++ b/python/setup.py @@ -352,61 +352,11 @@ def get_outputs(self): for name in self.get_names()] -# If the event of not running from a git clone (e.g. from a git archive -# or a Python sdist), see if we can set the version number ourselves -default_version = '17.0.0-SNAPSHOT' -if (not os.path.exists('../.git') and - not os.environ.get('SETUPTOOLS_SCM_PRETEND_VERSION')): - os.environ['SETUPTOOLS_SCM_PRETEND_VERSION'] = \ - default_version.replace('-SNAPSHOT', 'a0') - - -# See https://github.com/pypa/setuptools_scm#configuration-parameters -scm_version_write_to_prefix = os.environ.get( - 'SETUPTOOLS_SCM_VERSION_WRITE_TO_PREFIX', setup_dir) - - -def parse_git(root, **kwargs): - """ - Parse function for setuptools_scm that ignores tags for non-C++ - subprojects, e.g. apache-arrow-js-XXX tags. - """ - from setuptools_scm.git import parse - kwargs['describe_command'] =\ - 'git describe --dirty --tags --long --match "apache-arrow-[0-9]*.*"' - return parse(root, **kwargs) - - -def guess_next_dev_version(version): - if version.exact: - return version.format_with('{tag}') - else: - def guess_next_version(tag_version): - return default_version.replace('-SNAPSHOT', '') - return version.format_next_version(guess_next_version) - - -with open('README.md') as f: - long_description = f.read() - - class BinaryDistribution(Distribution): def has_ext_modules(foo): return True -install_requires = ( - 'numpy >= 1.16.6', -) - - -# Only include pytest-runner in setup_requires if we're invoking tests -if {'pytest', 'test', 'ptr'}.intersection(sys.argv): - setup_requires = ['pytest-runner'] -else: - setup_requires = [] - - if strtobool(os.environ.get('PYARROW_INSTALL_TESTS', '1')): packages = find_namespace_packages(include=['pyarrow*']) exclude_package_data = {} @@ -420,11 +370,7 @@ def has_ext_modules(foo): setup( - name='pyarrow', packages=packages, - zip_safe=False, - package_data={'pyarrow': ['*.pxd', '*.pyx', 'includes/*.pxd']}, - include_package_data=True, exclude_package_data=exclude_package_data, distclass=BinaryDistribution, # Dummy extension to trigger build_ext @@ -432,35 +378,4 @@ def has_ext_modules(foo): cmdclass={ 'build_ext': build_ext }, - use_scm_version={ - 'root': os.path.dirname(setup_dir), - 'parse': parse_git, - 'write_to': os.path.join(scm_version_write_to_prefix, - 'pyarrow/_generated_version.py'), - 'version_scheme': guess_next_dev_version - }, - setup_requires=['setuptools_scm', 'cython >= 0.29.31'] + setup_requires, - install_requires=install_requires, - tests_require=['pytest', 'pandas', 'hypothesis'], - python_requires='>=3.8', - description='Python library for Apache Arrow', - long_description=long_description, - long_description_content_type='text/markdown', - classifiers=[ - 'License :: OSI Approved :: Apache Software License', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - ], - license='Apache License, Version 2.0', - maintainer='Apache Arrow Developers', - maintainer_email='dev@arrow.apache.org', - test_suite='pyarrow.tests', - url='https://arrow.apache.org/', - project_urls={ - 'Documentation': 'https://arrow.apache.org/docs/python', - 'Source': 'https://github.com/apache/arrow', - }, ) From 5a8644156f55254e5852f94533bb5578b04d7a25 Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Wed, 5 Jun 2024 14:14:52 +0200 Subject: [PATCH 61/93] GH-40062: [C++] Move RecordBatch::ToTensor code from record_batch.cc to tensor.cc (#41932) ### Rationale for this change This is a precursor PR to https://github.com/apache/arrow/pull/41870 with the purpose to make the review of #41870 easier (the diff of the code will be visible as it currently isn't because the code was moved to table.cc. I should also live in tensor.cc). ### What changes are included in this PR? The code from `RecordBatch::ToTensor` in record_batch.cc is moved to `RecordBatchToTensor` in tensor.cc. ### Are these changes tested? Existing tests should pass. ### Are there any user-facing changes? No. **This PR does not close the linked issue yet, it is just a precursor!** * GitHub Issue: #40062 Authored-by: AlenkaF Signed-off-by: AlenkaF --- cpp/src/arrow/record_batch.cc | 198 +------------------------------- cpp/src/arrow/tensor.cc | 206 ++++++++++++++++++++++++++++++++++ cpp/src/arrow/tensor.h | 4 + 3 files changed, 212 insertions(+), 196 deletions(-) diff --git a/cpp/src/arrow/record_batch.cc b/cpp/src/arrow/record_batch.cc index 351f72f52365b..e3a8c0d710cb8 100644 --- a/cpp/src/arrow/record_batch.cc +++ b/cpp/src/arrow/record_batch.cc @@ -35,7 +35,6 @@ #include "arrow/type.h" #include "arrow/util/iterator.h" #include "arrow/util/logging.h" -#include "arrow/util/unreachable.h" #include "arrow/util/vector.h" #include "arrow/visit_type_inline.h" @@ -286,204 +285,11 @@ Result> RecordBatch::ToStructArray() const { /*offset=*/0); } -template -struct ConvertColumnsToTensorVisitor { - Out*& out_values; - const ArrayData& in_data; - - template - Status Visit(const T&) { - if constexpr (is_numeric(T::type_id)) { - using In = typename T::c_type; - auto in_values = ArraySpan(in_data).GetSpan(1, in_data.length); - - if (in_data.null_count == 0) { - if constexpr (std::is_same_v) { - memcpy(out_values, in_values.data(), in_values.size_bytes()); - out_values += in_values.size(); - } else { - for (In in_value : in_values) { - *out_values++ = static_cast(in_value); - } - } - } else { - for (int64_t i = 0; i < in_data.length; ++i) { - *out_values++ = - in_data.IsNull(i) ? static_cast(NAN) : static_cast(in_values[i]); - } - } - return Status::OK(); - } - Unreachable(); - } -}; - -template -struct ConvertColumnsToTensorRowMajorVisitor { - Out*& out_values; - const ArrayData& in_data; - int num_cols; - int col_idx; - - template - Status Visit(const T&) { - if constexpr (is_numeric(T::type_id)) { - using In = typename T::c_type; - auto in_values = ArraySpan(in_data).GetSpan(1, in_data.length); - - if (in_data.null_count == 0) { - for (int64_t i = 0; i < in_data.length; ++i) { - out_values[i * num_cols + col_idx] = static_cast(in_values[i]); - } - } else { - for (int64_t i = 0; i < in_data.length; ++i) { - out_values[i * num_cols + col_idx] = - in_data.IsNull(i) ? static_cast(NAN) : static_cast(in_values[i]); - } - } - return Status::OK(); - } - Unreachable(); - } -}; - -template -inline void ConvertColumnsToTensor(const RecordBatch& batch, uint8_t* out, - bool row_major) { - using CType = typename arrow::TypeTraits::CType; - auto* out_values = reinterpret_cast(out); - - int i = 0; - for (const auto& column : batch.columns()) { - if (row_major) { - ConvertColumnsToTensorRowMajorVisitor visitor{out_values, *column->data(), - batch.num_columns(), i++}; - DCHECK_OK(VisitTypeInline(*column->type(), &visitor)); - } else { - ConvertColumnsToTensorVisitor visitor{out_values, *column->data()}; - DCHECK_OK(VisitTypeInline(*column->type(), &visitor)); - } - } -} - Result> RecordBatch::ToTensor(bool null_to_nan, bool row_major, MemoryPool* pool) const { - if (num_columns() == 0) { - return Status::TypeError( - "Conversion to Tensor for RecordBatches without columns/schema is not " - "supported."); - } - // Check for no validity bitmap of each field - // if null_to_nan conversion is set to false - for (int i = 0; i < num_columns(); ++i) { - if (column(i)->null_count() > 0 && !null_to_nan) { - return Status::TypeError( - "Can only convert a RecordBatch with no nulls. Set null_to_nan to true to " - "convert nulls to NaN"); - } - } - - // Check for supported data types and merge fields - // to get the resulting uniform data type - if (!is_integer(column(0)->type()->id()) && !is_floating(column(0)->type()->id())) { - return Status::TypeError("DataType is not supported: ", - column(0)->type()->ToString()); - } - std::shared_ptr result_field = schema_->field(0); - std::shared_ptr result_type = result_field->type(); - - Field::MergeOptions options; - options.promote_integer_to_float = true; - options.promote_integer_sign = true; - options.promote_numeric_width = true; - - if (num_columns() > 1) { - for (int i = 1; i < num_columns(); ++i) { - if (!is_numeric(column(i)->type()->id())) { - return Status::TypeError("DataType is not supported: ", - column(i)->type()->ToString()); - } - - // Casting of float16 is not supported, throw an error in this case - if ((column(i)->type()->id() == Type::HALF_FLOAT || - result_field->type()->id() == Type::HALF_FLOAT) && - column(i)->type()->id() != result_field->type()->id()) { - return Status::NotImplemented("Casting from or to halffloat is not supported."); - } - - ARROW_ASSIGN_OR_RAISE( - result_field, result_field->MergeWith( - schema_->field(i)->WithName(result_field->name()), options)); - } - result_type = result_field->type(); - } - - // Check if result_type is signed or unsigned integer and null_to_nan is set to true - // Then all columns should be promoted to float type - if (is_integer(result_type->id()) && null_to_nan) { - ARROW_ASSIGN_OR_RAISE( - result_field, - result_field->MergeWith(field(result_field->name(), float32()), options)); - result_type = result_field->type(); - } - - // Allocate memory - ARROW_ASSIGN_OR_RAISE( - std::shared_ptr result, - AllocateBuffer(result_type->bit_width() * num_columns() * num_rows(), pool)); - // Copy data - switch (result_type->id()) { - case Type::UINT8: - ConvertColumnsToTensor(*this, result->mutable_data(), row_major); - break; - case Type::UINT16: - case Type::HALF_FLOAT: - ConvertColumnsToTensor(*this, result->mutable_data(), row_major); - break; - case Type::UINT32: - ConvertColumnsToTensor(*this, result->mutable_data(), row_major); - break; - case Type::UINT64: - ConvertColumnsToTensor(*this, result->mutable_data(), row_major); - break; - case Type::INT8: - ConvertColumnsToTensor(*this, result->mutable_data(), row_major); - break; - case Type::INT16: - ConvertColumnsToTensor(*this, result->mutable_data(), row_major); - break; - case Type::INT32: - ConvertColumnsToTensor(*this, result->mutable_data(), row_major); - break; - case Type::INT64: - ConvertColumnsToTensor(*this, result->mutable_data(), row_major); - break; - case Type::FLOAT: - ConvertColumnsToTensor(*this, result->mutable_data(), row_major); - break; - case Type::DOUBLE: - ConvertColumnsToTensor(*this, result->mutable_data(), row_major); - break; - default: - return Status::TypeError("DataType is not supported: ", result_type->ToString()); - } - - // Construct Tensor object - const auto& fixed_width_type = - internal::checked_cast(*result_type); - std::vector shape = {num_rows(), num_columns()}; - std::vector strides; std::shared_ptr tensor; - - if (row_major) { - ARROW_RETURN_NOT_OK( - internal::ComputeRowMajorStrides(fixed_width_type, shape, &strides)); - } else { - ARROW_RETURN_NOT_OK( - internal::ComputeColumnMajorStrides(fixed_width_type, shape, &strides)); - } - ARROW_ASSIGN_OR_RAISE(tensor, - Tensor::Make(result_type, std::move(result), shape, strides)); + ARROW_RETURN_NOT_OK( + internal::RecordBatchToTensor(*this, null_to_nan, row_major, pool, &tensor)); return tensor; } diff --git a/cpp/src/arrow/tensor.cc b/cpp/src/arrow/tensor.cc index 77ccedbde15c6..b47f1a1075b37 100644 --- a/cpp/src/arrow/tensor.cc +++ b/cpp/src/arrow/tensor.cc @@ -18,6 +18,7 @@ #include "arrow/tensor.h" #include +#include #include #include #include @@ -27,12 +28,14 @@ #include #include +#include "arrow/record_batch.h" #include "arrow/status.h" #include "arrow/type.h" #include "arrow/type_traits.h" #include "arrow/util/checked_cast.h" #include "arrow/util/int_util_overflow.h" #include "arrow/util/logging.h" +#include "arrow/util/unreachable.h" #include "arrow/visit_type_inline.h" namespace arrow { @@ -220,6 +223,209 @@ Status ValidateTensorParameters(const std::shared_ptr& type, return Status::OK(); } +template +struct ConvertColumnsToTensorVisitor { + Out*& out_values; + const ArrayData& in_data; + + template + Status Visit(const T&) { + if constexpr (is_numeric(T::type_id)) { + using In = typename T::c_type; + auto in_values = ArraySpan(in_data).GetSpan(1, in_data.length); + + if (in_data.null_count == 0) { + if constexpr (std::is_same_v) { + memcpy(out_values, in_values.data(), in_values.size_bytes()); + out_values += in_values.size(); + } else { + for (In in_value : in_values) { + *out_values++ = static_cast(in_value); + } + } + } else { + for (int64_t i = 0; i < in_data.length; ++i) { + *out_values++ = + in_data.IsNull(i) ? static_cast(NAN) : static_cast(in_values[i]); + } + } + return Status::OK(); + } + Unreachable(); + } +}; + +template +struct ConvertColumnsToTensorRowMajorVisitor { + Out*& out_values; + const ArrayData& in_data; + int num_cols; + int col_idx; + + template + Status Visit(const T&) { + if constexpr (is_numeric(T::type_id)) { + using In = typename T::c_type; + auto in_values = ArraySpan(in_data).GetSpan(1, in_data.length); + + if (in_data.null_count == 0) { + for (int64_t i = 0; i < in_data.length; ++i) { + out_values[i * num_cols + col_idx] = static_cast(in_values[i]); + } + } else { + for (int64_t i = 0; i < in_data.length; ++i) { + out_values[i * num_cols + col_idx] = + in_data.IsNull(i) ? static_cast(NAN) : static_cast(in_values[i]); + } + } + return Status::OK(); + } + Unreachable(); + } +}; + +template +inline void ConvertColumnsToTensor(const RecordBatch& batch, uint8_t* out, + bool row_major) { + using CType = typename arrow::TypeTraits::CType; + auto* out_values = reinterpret_cast(out); + + int i = 0; + for (const auto& column : batch.columns()) { + if (row_major) { + ConvertColumnsToTensorRowMajorVisitor visitor{out_values, *column->data(), + batch.num_columns(), i++}; + DCHECK_OK(VisitTypeInline(*column->type(), &visitor)); + } else { + ConvertColumnsToTensorVisitor visitor{out_values, *column->data()}; + DCHECK_OK(VisitTypeInline(*column->type(), &visitor)); + } + } +} + +Status RecordBatchToTensor(const RecordBatch& batch, bool null_to_nan, bool row_major, + MemoryPool* pool, std::shared_ptr* tensor) { + if (batch.num_columns() == 0) { + return Status::TypeError( + "Conversion to Tensor for RecordBatches without columns/schema is not " + "supported."); + } + // Check for no validity bitmap of each field + // if null_to_nan conversion is set to false + for (int i = 0; i < batch.num_columns(); ++i) { + if (batch.column(i)->null_count() > 0 && !null_to_nan) { + return Status::TypeError( + "Can only convert a RecordBatch with no nulls. Set null_to_nan to true to " + "convert nulls to NaN"); + } + } + + // Check for supported data types and merge fields + // to get the resulting uniform data type + if (!is_integer(batch.column(0)->type()->id()) && + !is_floating(batch.column(0)->type()->id())) { + return Status::TypeError("DataType is not supported: ", + batch.column(0)->type()->ToString()); + } + std::shared_ptr result_field = batch.schema()->field(0); + std::shared_ptr result_type = result_field->type(); + + Field::MergeOptions options; + options.promote_integer_to_float = true; + options.promote_integer_sign = true; + options.promote_numeric_width = true; + + if (batch.num_columns() > 1) { + for (int i = 1; i < batch.num_columns(); ++i) { + if (!is_numeric(batch.column(i)->type()->id())) { + return Status::TypeError("DataType is not supported: ", + batch.column(i)->type()->ToString()); + } + + // Casting of float16 is not supported, throw an error in this case + if ((batch.column(i)->type()->id() == Type::HALF_FLOAT || + result_field->type()->id() == Type::HALF_FLOAT) && + batch.column(i)->type()->id() != result_field->type()->id()) { + return Status::NotImplemented("Casting from or to halffloat is not supported."); + } + + ARROW_ASSIGN_OR_RAISE( + result_field, + result_field->MergeWith( + batch.schema()->field(i)->WithName(result_field->name()), options)); + } + result_type = result_field->type(); + } + + // Check if result_type is signed or unsigned integer and null_to_nan is set to true + // Then all columns should be promoted to float type + if (is_integer(result_type->id()) && null_to_nan) { + ARROW_ASSIGN_OR_RAISE( + result_field, + result_field->MergeWith(field(result_field->name(), float32()), options)); + result_type = result_field->type(); + } + + // Allocate memory + ARROW_ASSIGN_OR_RAISE( + std::shared_ptr result, + AllocateBuffer(result_type->bit_width() * batch.num_columns() * batch.num_rows(), + pool)); + // Copy data + switch (result_type->id()) { + case Type::UINT8: + ConvertColumnsToTensor(batch, result->mutable_data(), row_major); + break; + case Type::UINT16: + case Type::HALF_FLOAT: + ConvertColumnsToTensor(batch, result->mutable_data(), row_major); + break; + case Type::UINT32: + ConvertColumnsToTensor(batch, result->mutable_data(), row_major); + break; + case Type::UINT64: + ConvertColumnsToTensor(batch, result->mutable_data(), row_major); + break; + case Type::INT8: + ConvertColumnsToTensor(batch, result->mutable_data(), row_major); + break; + case Type::INT16: + ConvertColumnsToTensor(batch, result->mutable_data(), row_major); + break; + case Type::INT32: + ConvertColumnsToTensor(batch, result->mutable_data(), row_major); + break; + case Type::INT64: + ConvertColumnsToTensor(batch, result->mutable_data(), row_major); + break; + case Type::FLOAT: + ConvertColumnsToTensor(batch, result->mutable_data(), row_major); + break; + case Type::DOUBLE: + ConvertColumnsToTensor(batch, result->mutable_data(), row_major); + break; + default: + return Status::TypeError("DataType is not supported: ", result_type->ToString()); + } + + // Construct Tensor object + const auto& fixed_width_type = + internal::checked_cast(*result_type); + std::vector shape = {batch.num_rows(), batch.num_columns()}; + std::vector strides; + + if (row_major) { + ARROW_RETURN_NOT_OK( + internal::ComputeRowMajorStrides(fixed_width_type, shape, &strides)); + } else { + ARROW_RETURN_NOT_OK( + internal::ComputeColumnMajorStrides(fixed_width_type, shape, &strides)); + } + ARROW_ASSIGN_OR_RAISE(*tensor, + Tensor::Make(result_type, std::move(result), shape, strides)); + return Status::OK(); +} + } // namespace internal /// Constructor with strides and dimension names diff --git a/cpp/src/arrow/tensor.h b/cpp/src/arrow/tensor.h index ff6f3735f9193..dd3a21fae495a 100644 --- a/cpp/src/arrow/tensor.h +++ b/cpp/src/arrow/tensor.h @@ -77,6 +77,10 @@ Status ValidateTensorParameters(const std::shared_ptr& type, const std::vector& strides, const std::vector& dim_names); +ARROW_EXPORT +Status RecordBatchToTensor(const RecordBatch& batch, bool null_to_nan, bool row_major, + MemoryPool* pool, std::shared_ptr* tensor); + } // namespace internal class ARROW_EXPORT Tensor { From 37d0acdccbf1228574434499ccb3a63d7a09e16f Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 5 Jun 2024 16:28:27 +0200 Subject: [PATCH 62/93] GH-41983: [Dev] Run issue labeling bot only when opening an issue (not editing) (#41986) ### Rationale for this change Currently the bot will remove manually added Component labels, because at that point you are editing the issue and the workflow is run again, reinstating the labels in the "Components" section in the top post created by the issue form. Therefore, restrict this bot to only run when the issue is "opened" * GitHub Issue: #41983 Authored-by: Joris Van den Bossche Signed-off-by: Jacob Wujciak-Jens --- .github/workflows/issue_bot.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/issue_bot.yml b/.github/workflows/issue_bot.yml index ec614ca1e7c56..2725825b56952 100644 --- a/.github/workflows/issue_bot.yml +++ b/.github/workflows/issue_bot.yml @@ -21,7 +21,6 @@ on: issues: types: - opened - - edited permissions: contents: read From 0b5f0a2af191078cb86d467035b1f19560e2e93a Mon Sep 17 00:00:00 2001 From: Jaap Versteegh Date: Wed, 5 Jun 2024 16:53:56 +0200 Subject: [PATCH 63/93] GH-41502: [Python] Fix reading column index with decimal values (#41503) Fix for #41502 Convert pandas "decimal" to "object" in numpy. * GitHub Issue: #41502 Authored-by: Jaap Versteegh Signed-off-by: Joris Van den Bossche --- python/pyarrow/pandas_compat.py | 5 +++++ python/pyarrow/tests/test_pandas.py | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py index 00fa19604e5c3..e246f1263d20d 100644 --- a/python/pyarrow/pandas_compat.py +++ b/python/pyarrow/pandas_compat.py @@ -23,6 +23,7 @@ # module bug (ARROW-11983) import concurrent.futures.thread # noqa from copy import deepcopy +import decimal from itertools import zip_longest import json import operator @@ -1027,6 +1028,7 @@ def _is_generated_index_name(name): 'string': np.str_, 'integer': np.int64, 'floating': np.float64, + 'decimal': np.object_, 'empty': np.object_, } @@ -1105,6 +1107,9 @@ def _reconstruct_columns_from_metadata(columns, column_indexes): tz = pa.lib.string_to_tzinfo( column_indexes[0]['metadata']['timezone']) level = pd.to_datetime(level, utc=True).tz_convert(tz) + # GH-41503: if the column index was decimal, restore to decimal + elif pandas_dtype == "decimal": + level = _pandas_api.pd.Index([decimal.Decimal(i) for i in level]) elif level.dtype != dtype: level = level.astype(dtype) # ARROW-9096: if original DataFrame was upcast we keep that diff --git a/python/pyarrow/tests/test_pandas.py b/python/pyarrow/tests/test_pandas.py index 3678b4e57a9a8..be2c5b14e68b0 100644 --- a/python/pyarrow/tests/test_pandas.py +++ b/python/pyarrow/tests/test_pandas.py @@ -221,6 +221,17 @@ def test_column_index_names_with_tz(self): ) _check_pandas_roundtrip(df, preserve_index=True) + def test_column_index_names_with_decimal(self): + # GH-41503: Test valid roundtrip with decimal value in column index + df = pd.DataFrame( + [[decimal.Decimal(5), decimal.Decimal(6)]], + columns=pd.MultiIndex.from_product( + [[decimal.Decimal(1)], [decimal.Decimal(2), decimal.Decimal(3)]] + ), + index=[decimal.Decimal(4)], + ) + _check_pandas_roundtrip(df, preserve_index=True) + def test_range_index_shortcut(self): # ARROW-1639 index_name = 'foo' From cd7ebc0b47668339f315b4ba224ce271c46c6cf5 Mon Sep 17 00:00:00 2001 From: mwish Date: Wed, 5 Jun 2024 23:21:14 +0800 Subject: [PATCH 64/93] GH-41953: [C++] Minor enhance code style for FixedShapeTensorType (#41954) ### Rationale for this change Minor enhance code style for FixedShapeTensorType ### What changes are included in this PR? 1. Remove some `shared_ptr` temp variables 2. Some interfaces allowing return reference ### Are these changes tested? Covered by existing ### Are there any user-facing changes? no * GitHub Issue: #41953 Authored-by: mwish Signed-off-by: Antoine Pitrou --- cpp/src/arrow/extension/fixed_shape_tensor.cc | 66 ++++++++++--------- cpp/src/arrow/extension/fixed_shape_tensor.h | 4 +- 2 files changed, 36 insertions(+), 34 deletions(-) diff --git a/cpp/src/arrow/extension/fixed_shape_tensor.cc b/cpp/src/arrow/extension/fixed_shape_tensor.cc index 1101b08307332..944a134a707b1 100644 --- a/cpp/src/arrow/extension/fixed_shape_tensor.cc +++ b/cpp/src/arrow/extension/fixed_shape_tensor.cc @@ -207,44 +207,44 @@ std::shared_ptr FixedShapeTensorType::MakeArray( Result> FixedShapeTensorType::MakeTensor( const std::shared_ptr& scalar) { - const auto ext_scalar = internal::checked_pointer_cast(scalar); - const auto ext_type = - internal::checked_pointer_cast(scalar->type); - if (!is_fixed_width(*ext_type->value_type())) { + const auto& ext_scalar = internal::checked_cast(*scalar); + const auto& ext_type = + internal::checked_cast(*scalar->type); + if (!is_fixed_width(*ext_type.value_type())) { return Status::TypeError("Cannot convert non-fixed-width values to Tensor."); } - const auto array = - internal::checked_pointer_cast(ext_scalar->value)->value; + const auto& array = + internal::checked_cast(ext_scalar.value.get())->value; if (array->null_count() > 0) { return Status::Invalid("Cannot convert data with nulls to Tensor."); } - const auto value_type = - internal::checked_pointer_cast(ext_type->value_type()); - const auto byte_width = value_type->byte_width(); + const auto& value_type = + internal::checked_cast(*ext_type.value_type()); + const auto byte_width = value_type.byte_width(); - std::vector permutation = ext_type->permutation(); + std::vector permutation = ext_type.permutation(); if (permutation.empty()) { - permutation.resize(ext_type->ndim()); + permutation.resize(ext_type.ndim()); std::iota(permutation.begin(), permutation.end(), 0); } - std::vector shape = ext_type->shape(); + std::vector shape = ext_type.shape(); internal::Permute(permutation, &shape); - std::vector dim_names = ext_type->dim_names(); + std::vector dim_names = ext_type.dim_names(); if (!dim_names.empty()) { internal::Permute(permutation, &dim_names); } std::vector strides; - RETURN_NOT_OK(ComputeStrides(*value_type.get(), shape, permutation, &strides)); + RETURN_NOT_OK(ComputeStrides(value_type, shape, permutation, &strides)); const auto start_position = array->offset() * byte_width; const auto size = std::accumulate(shape.begin(), shape.end(), static_cast(1), std::multiplies<>()); const auto buffer = SliceBuffer(array->data()->buffers[1], start_position, size * byte_width); - return Tensor::Make(ext_type->value_type(), buffer, shape, strides, dim_names); + return Tensor::Make(ext_type.value_type(), buffer, shape, strides, dim_names); } Result> FixedShapeTensorArray::FromTensor( @@ -257,12 +257,14 @@ Result> FixedShapeTensorArray::FromTensor permutation.erase(permutation.begin()); std::vector cell_shape; + cell_shape.reserve(permutation.size()); for (auto i : permutation) { cell_shape.emplace_back(tensor->shape()[i]); } std::vector dim_names; if (!tensor->dim_names().empty()) { + dim_names.reserve(permutation.size()); for (auto i : permutation) { dim_names.emplace_back(tensor->dim_names()[i]); } @@ -337,9 +339,9 @@ const Result> FixedShapeTensorArray::ToTensor() const { // To convert an array of n dimensional tensors to a n+1 dimensional tensor we // interpret the array's length as the first dimension the new tensor. - const auto ext_type = - internal::checked_pointer_cast(this->type()); - const auto value_type = ext_type->value_type(); + const auto& ext_type = + internal::checked_cast(*this->type()); + const auto& value_type = ext_type.value_type(); ARROW_RETURN_IF( !is_fixed_width(*value_type), Status::TypeError(value_type->ToString(), " is not valid data type for a tensor")); @@ -350,35 +352,35 @@ const Result> FixedShapeTensorArray::ToTensor() const { // will get permutation index 0 and remaining values from ext_type->permutation() need // to be shifted to fill the [1, ndim+1) range. Computed permutation will be used to // generate the new tensor's shape, strides and dim_names. - std::vector permutation = ext_type->permutation(); + std::vector permutation = ext_type.permutation(); if (permutation.empty()) { - permutation.resize(ext_type->ndim() + 1); + permutation.resize(ext_type.ndim() + 1); std::iota(permutation.begin(), permutation.end(), 0); } else { - for (auto i = 0; i < static_cast(ext_type->ndim()); i++) { + for (auto i = 0; i < static_cast(ext_type.ndim()); i++) { permutation[i] += 1; } permutation.insert(permutation.begin(), 1, 0); } - std::vector dim_names = ext_type->dim_names(); + std::vector dim_names = ext_type.dim_names(); if (!dim_names.empty()) { dim_names.insert(dim_names.begin(), 1, ""); internal::Permute(permutation, &dim_names); } - std::vector shape = ext_type->shape(); + std::vector shape = ext_type.shape(); auto cell_size = std::accumulate(shape.begin(), shape.end(), static_cast(1), std::multiplies<>()); shape.insert(shape.begin(), 1, this->length()); internal::Permute(permutation, &shape); std::vector tensor_strides; - const auto fw_value_type = internal::checked_pointer_cast(value_type); + const auto* fw_value_type = internal::checked_cast(value_type.get()); ARROW_RETURN_NOT_OK( - ComputeStrides(*fw_value_type.get(), shape, permutation, &tensor_strides)); + ComputeStrides(*fw_value_type, shape, permutation, &tensor_strides)); - const auto raw_buffer = this->storage()->data()->child_data[0]->buffers[1]; + const auto& raw_buffer = this->storage()->data()->child_data[0]->buffers[1]; ARROW_ASSIGN_OR_RAISE( const auto buffer, SliceBufferSafe(raw_buffer, this->offset() * cell_size * value_type->byte_width())); @@ -389,7 +391,7 @@ const Result> FixedShapeTensorArray::ToTensor() const { Result> FixedShapeTensorType::Make( const std::shared_ptr& value_type, const std::vector& shape, const std::vector& permutation, const std::vector& dim_names) { - const auto ndim = shape.size(); + const size_t ndim = shape.size(); if (!permutation.empty() && ndim != permutation.size()) { return Status::Invalid("permutation size must match shape size. Expected: ", ndim, " Got: ", permutation.size()); @@ -402,18 +404,18 @@ Result> FixedShapeTensorType::Make( RETURN_NOT_OK(internal::IsPermutationValid(permutation)); } - const auto size = std::accumulate(shape.begin(), shape.end(), static_cast(1), - std::multiplies<>()); + const int64_t size = std::accumulate(shape.begin(), shape.end(), + static_cast(1), std::multiplies<>()); return std::make_shared(value_type, static_cast(size), shape, permutation, dim_names); } const std::vector& FixedShapeTensorType::strides() { if (strides_.empty()) { - auto value_type = internal::checked_pointer_cast(this->value_type_); + auto value_type = internal::checked_cast(this->value_type_.get()); std::vector tensor_strides; - ARROW_CHECK_OK(ComputeStrides(*value_type.get(), this->shape(), this->permutation(), - &tensor_strides)); + ARROW_CHECK_OK( + ComputeStrides(*value_type, this->shape(), this->permutation(), &tensor_strides)); strides_ = tensor_strides; } return strides_; diff --git a/cpp/src/arrow/extension/fixed_shape_tensor.h b/cpp/src/arrow/extension/fixed_shape_tensor.h index 3fec79b5c2a3c..20ec20a64c2d4 100644 --- a/cpp/src/arrow/extension/fixed_shape_tensor.h +++ b/cpp/src/arrow/extension/fixed_shape_tensor.h @@ -67,10 +67,10 @@ class ARROW_EXPORT FixedShapeTensorType : public ExtensionType { size_t ndim() const { return shape_.size(); } /// Shape of tensor elements - const std::vector shape() const { return shape_; } + const std::vector& shape() const { return shape_; } /// Value type of tensor elements - const std::shared_ptr value_type() const { return value_type_; } + const std::shared_ptr& value_type() const { return value_type_; } /// Strides of tensor elements. Strides state offset in bytes between adjacent /// elements along each dimension. In case permutation is non-empty strides are From 51bc2a61c90a89a29dacacbada190aa06f232271 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 5 Jun 2024 17:57:30 +0200 Subject: [PATCH 65/93] GH-41797: [C++][S3] Remove GetBucketRegion hack for newer AWS SDK versions (#41798) ### Rationale for this change To get the region a S3 bucket resides on, it is required to issue a HeadBucket request and parse the response headers for a certain header value. Unfortunately, the AWS SDK doesn't let us access arbitrary headers on successful responses for S3 model requests, which had us implement a workaround by calling lower-level SDK APIs. However, the SDK recently added a `GetBucketRegion` method on `HeadBucketRequest`, which obsoletes the need for this workaround. We now use this method if the available AWS SDK version is recent enough. ### Are these changes tested? By existing tests on the various CI configurations. ### Are there any user-facing changes? No. * GitHub Issue: #41797 Authored-by: Antoine Pitrou Signed-off-by: Antoine Pitrou --- cpp/src/arrow/filesystem/s3fs.cc | 81 +++++++++++++++++++++++--------- 1 file changed, 59 insertions(+), 22 deletions(-) diff --git a/cpp/src/arrow/filesystem/s3fs.cc b/cpp/src/arrow/filesystem/s3fs.cc index 78e02c31a35a3..c456be2d0d3cd 100644 --- a/cpp/src/arrow/filesystem/s3fs.cc +++ b/cpp/src/arrow/filesystem/s3fs.cc @@ -601,44 +601,81 @@ class S3Client : public Aws::S3::S3Client { public: using Aws::S3::S3Client::S3Client; + static inline constexpr auto kBucketRegionHeaderName = "x-amz-bucket-region"; + + std::string GetBucketRegionFromHeaders( + const Aws::Http::HeaderValueCollection& headers) { + const auto it = headers.find(ToAwsString(kBucketRegionHeaderName)); + if (it != headers.end()) { + return std::string(FromAwsString(it->second)); + } + return std::string(); + } + + template + Result GetBucketRegionFromError( + const std::string& bucket, const Aws::Client::AWSError& error) { + std::string region = GetBucketRegionFromHeaders(error.GetResponseHeaders()); + if (!region.empty()) { + return region; + } else if (error.GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { + return Status::IOError("Bucket '", bucket, "' not found"); + } else { + return ErrorToStatus( + std::forward_as_tuple("When resolving region for bucket '", bucket, "': "), + "HeadBucket", error); + } + } + +#if ARROW_AWS_SDK_VERSION_CHECK(1, 11, 212) + // HeadBucketResult::GetBucketRegion appeared in AWS SDK 1.11.212 + Result GetBucketRegion(const std::string& bucket, + const S3Model::HeadBucketRequest& request) { + auto outcome = this->HeadBucket(request); + if (!outcome.IsSuccess()) { + return GetBucketRegionFromError(bucket, outcome.GetError()); + } + auto&& region = std::move(outcome).GetResult().GetBucketRegion(); + if (region.empty()) { + return Status::IOError("When resolving region for bucket '", request.GetBucket(), + "': missing 'x-amz-bucket-region' header in response"); + } + return region; + } +#else // To get a bucket's region, we must extract the "x-amz-bucket-region" header // from the response to a HEAD bucket request. // Unfortunately, the S3Client APIs don't let us access the headers of successful // responses. So we have to cook a AWS request and issue it ourselves. - - Result GetBucketRegion(const S3Model::HeadBucketRequest& request) { + Result GetBucketRegion(const std::string& bucket, + const S3Model::HeadBucketRequest& request) { auto uri = GeneratePresignedUrl(request.GetBucket(), /*key=*/"", Aws::Http::HttpMethod::HTTP_HEAD); // NOTE: The signer region argument isn't passed here, as there's no easy // way of computing it (the relevant method is private). auto outcome = MakeRequest(uri, request, Aws::Http::HttpMethod::HTTP_HEAD, Aws::Auth::SIGV4_SIGNER); - const auto code = outcome.IsSuccess() ? outcome.GetResult().GetResponseCode() - : outcome.GetError().GetResponseCode(); - const auto& headers = outcome.IsSuccess() - ? outcome.GetResult().GetHeaderValueCollection() - : outcome.GetError().GetResponseHeaders(); - - const auto it = headers.find(ToAwsString("x-amz-bucket-region")); - if (it == headers.end()) { - if (code == Aws::Http::HttpResponseCode::NOT_FOUND) { - return Status::IOError("Bucket '", request.GetBucket(), "' not found"); - } else if (!outcome.IsSuccess()) { - return ErrorToStatus(std::forward_as_tuple("When resolving region for bucket '", - request.GetBucket(), "': "), - "HeadBucket", outcome.GetError()); - } else { - return Status::IOError("When resolving region for bucket '", request.GetBucket(), - "': missing 'x-amz-bucket-region' header in response"); - } + if (!outcome.IsSuccess()) { + return GetBucketRegionFromError(bucket, outcome.GetError()); + } + std::string region = + GetBucketRegionFromHeaders(outcome.GetResult().GetHeaderValueCollection()); + if (!region.empty()) { + return region; + } else if (outcome.GetResult().GetResponseCode() == + Aws::Http::HttpResponseCode::NOT_FOUND) { + return Status::IOError("Bucket '", request.GetBucket(), "' not found"); + } else { + return Status::IOError("When resolving region for bucket '", request.GetBucket(), + "': missing 'x-amz-bucket-region' header in response"); } - return std::string(FromAwsString(it->second)); } +#endif Result GetBucketRegion(const std::string& bucket) { S3Model::HeadBucketRequest req; req.SetBucket(ToAwsString(bucket)); - return GetBucketRegion(req); + return GetBucketRegion(bucket, req); } S3Model::CompleteMultipartUploadOutcome CompleteMultipartUploadWithErrorFixup( From 9ee6ea701e20d1b47934f977d87811624061d597 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 5 Jun 2024 19:06:46 +0200 Subject: [PATCH 66/93] MINOR: [C++][CI] Work around bug in conda-forge benchmark package (#41987) ### Rationale for this change Work around bug in version 1.8.4 of the benchmark package: https://github.com/conda-forge/benchmark-feedstock/issues/36 ### Are these changes tested? By regular CI jobs. ### Are there any user-facing changes? No. Authored-by: Antoine Pitrou Signed-off-by: Jacob Wujciak-Jens --- .github/workflows/cpp.yml | 2 ++ ci/conda_env_cpp.txt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index dd5abbe1b4b1b..e539fadb859fe 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -21,6 +21,7 @@ on: push: paths: - '.github/workflows/cpp.yml' + - 'ci/conda_env_*' - 'ci/docker/**' - 'ci/scripts/cpp_*' - 'ci/scripts/install_azurite.sh' @@ -35,6 +36,7 @@ on: pull_request: paths: - '.github/workflows/cpp.yml' + - 'ci/conda_env_*' - 'ci/docker/**' - 'ci/scripts/cpp_*' - 'ci/scripts/install_azurite.sh' diff --git a/ci/conda_env_cpp.txt b/ci/conda_env_cpp.txt index 52e456eaab0cc..f28a24cac8d2d 100644 --- a/ci/conda_env_cpp.txt +++ b/ci/conda_env_cpp.txt @@ -21,7 +21,7 @@ azure-identity-cpp>=1.6.0 azure-storage-blobs-cpp>=12.10.0 azure-storage-common-cpp>=12.5.0 azure-storage-files-datalake-cpp>=12.9.0 -benchmark>=1.6.0 +benchmark>=1.6.0,!=1.8.4 boost-cpp>=1.68.0 brotli bzip2 From 3d1120551737787b3e2008389b67b451a045fb10 Mon Sep 17 00:00:00 2001 From: abandy Date: Wed, 5 Jun 2024 22:58:04 -0400 Subject: [PATCH 67/93] GH-41999: [Swift] Add methods for adding array and vargs to arrow array (#42000) ### Rationale for this change Would be nice to have methods for adding an array of values or variable args to when constructing an arrow array. ### Are these changes tested? Yes, tests are included * GitHub Issue: #41999 Authored-by: Alva Bandy Signed-off-by: Sutou Kouhei --- .../Sources/Arrow/ArrowArrayBuilder.swift | 12 ++++++ swift/Arrow/Tests/ArrowTests/ArrayTests.swift | 40 ++++++++++++++++++- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/swift/Arrow/Sources/Arrow/ArrowArrayBuilder.swift b/swift/Arrow/Sources/Arrow/ArrowArrayBuilder.swift index 4865b8a791256..40f9628d8f162 100644 --- a/swift/Arrow/Sources/Arrow/ArrowArrayBuilder.swift +++ b/swift/Arrow/Sources/Arrow/ArrowArrayBuilder.swift @@ -35,6 +35,18 @@ public class ArrowArrayBuilder> self.bufferBuilder = try T() } + public func append(_ vals: T.ItemType?...) { + for val in vals { + self.bufferBuilder.append(val) + } + } + + public func append(_ vals: [T.ItemType?]) { + for val in vals { + self.bufferBuilder.append(val) + } + } + public func append(_ val: T.ItemType?) { self.bufferBuilder.append(val) } diff --git a/swift/Arrow/Tests/ArrowTests/ArrayTests.swift b/swift/Arrow/Tests/ArrowTests/ArrayTests.swift index 10ffc4f96d83e..ed0cb1148e871 100644 --- a/swift/Arrow/Tests/ArrowTests/ArrayTests.swift +++ b/swift/Arrow/Tests/ArrowTests/ArrayTests.swift @@ -18,7 +18,7 @@ import XCTest @testable import Arrow -final class ArrayTests: XCTestCase { +final class ArrayTests: XCTestCase { // swiftlint:disable:this type_body_length func testPrimitiveArray() throws { // This is an example of a functional test case. // Use XCTAssert and related functions to verify your tests produce the correct @@ -271,4 +271,40 @@ final class ArrayTests: XCTestCase { XCTAssertEqual(stringHolder.nullCount, 10) XCTAssertEqual(stringHolder.length, 100) } - } + + func testAddVArgs() throws { + let arrayBuilder: NumberArrayBuilder = try ArrowArrayBuilders.loadNumberArrayBuilder() + arrayBuilder.append(0, 1, 2, 3, 4, 5, 6, 7, 8, 9) + XCTAssertEqual(arrayBuilder.length, 10) + XCTAssertEqual(try arrayBuilder.finish()[2], 2) + let doubleBuilder: NumberArrayBuilder = try ArrowArrayBuilders.loadNumberArrayBuilder() + doubleBuilder.append(0, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8) + XCTAssertEqual(doubleBuilder.length, 9) + XCTAssertEqual(try doubleBuilder.finish()[4], 4.4) + let stringBuilder = try ArrowArrayBuilders.loadStringArrayBuilder() + stringBuilder.append("0", "1", "2", "3", "4", "5", "6") + XCTAssertEqual(stringBuilder.length, 7) + XCTAssertEqual(try stringBuilder.finish()[4], "4") + let boolBuilder = try ArrowArrayBuilders.loadBoolArrayBuilder() + boolBuilder.append(true, false, true, false) + XCTAssertEqual(try boolBuilder.finish()[2], true) + } + + func testAddArray() throws { + let arrayBuilder: NumberArrayBuilder = try ArrowArrayBuilders.loadNumberArrayBuilder() + arrayBuilder.append([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + XCTAssertEqual(arrayBuilder.length, 10) + XCTAssertEqual(try arrayBuilder.finish()[2], 2) + let doubleBuilder: NumberArrayBuilder = try ArrowArrayBuilders.loadNumberArrayBuilder() + doubleBuilder.append([0, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8]) + XCTAssertEqual(doubleBuilder.length, 9) + XCTAssertEqual(try doubleBuilder.finish()[4], 4.4) + let stringBuilder = try ArrowArrayBuilders.loadStringArrayBuilder() + stringBuilder.append(["0", "1", "2", "3", "4", "5", "6"]) + XCTAssertEqual(stringBuilder.length, 7) + XCTAssertEqual(try stringBuilder.finish()[4], "4") + let boolBuilder = try ArrowArrayBuilders.loadBoolArrayBuilder() + boolBuilder.append([true, false, true, false]) + XCTAssertEqual(try boolBuilder.finish()[2], true) + } +} From dbcce63b925118bb1ba35455a684a21620a250ac Mon Sep 17 00:00:00 2001 From: Vibhatha Lakmal Abeykoon Date: Thu, 6 Jun 2024 10:42:47 +0530 Subject: [PATCH 68/93] MINOR: [Java] Bump com.google.errorprone:error_prone_core from 2.10.0 to 2.28.0 in /java - Manual Fix (#41996) ### Rationale for this change Based on the [dependabot PR ](https://github.com/apache/arrow/pull/41944), making correct amendments to reflect the necessary error-prone library upgrade. ### What changes are included in this PR? - [X] Upgrade error-prone version for JDK 11+ - [X] Keeping the existing error-prone version for JDK 8, and added notes and references as reasoning. ### Are these changes tested? Tested by existing build. ### Are there any user-facing changes? No Authored-by: Vibhatha Abeykoon Signed-off-by: David Li --- java/pom.xml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index a59d29c576398..0e9b7f0e25a34 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -94,9 +94,7 @@ 2 true 9+181-r4173-1 - 2.24.0 - 2.24.0 - 2.10.0 + 2.28.0 3.12.1 5.11.0 5.2.0 @@ -946,6 +944,9 @@ Do not activate Error Prone while running with Eclipse/M2E as it causes incompatibilities with other annotation processors. See https://github.com/jbosstools/m2e-apt/issues/62 for details + + Note: Maintaining error-prone version with JDK8 + See https://github.com/google/error-prone/blob/f8e33bc460be82ab22256a7ef8b979d7a2cacaba/docs/installation.md#jdk-8 --> 1.8 @@ -969,6 +970,13 @@ com.google.errorprone error_prone_core + 2.10.0 From 41810749e086278aea541240a16e9cf1e32eab80 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Jun 2024 16:15:52 +0900 Subject: [PATCH 69/93] MINOR: [Java] Bump org.apache.calcite.avatica:avatica from 1.24.0 to 1.25.0 in /java (#41212) Bumps [org.apache.calcite.avatica:avatica](https://github.com/apache/calcite-avatica) from 1.24.0 to 1.25.0.
Commits
  • 62b0fdd [CALCITE-6334] Release Avatica 1.25.0
  • c0cb4b7 [CALCITE-6248] Illegal dates are accepted by casts
  • 8c36e01 [CALCITE-6282] Avatica ignores time precision when returning TIME results
  • ef9a5a6 [CALCITE-6137] Upgrade Gradle from 8.1.1 to 8.4, support jdk21
  • c12c3a3 Apply same vcs.xml as for Calcite
  • 4b9c823 [CALCITE-6209] Long queries are failing with "java.net.SocketTimeoutException...
  • 275a082 [CALCITE-6280] Jetty version number leaked by Avatica http server
  • bc7ba9e Disable JIRA worklog notifications for GitHub PRs
  • 178ff82 Add Calcite CLI tool to list of Avatica Clients on website
  • 07c6b8d [CALCITE-6212] Config locale = en_US for javadoc task
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.apache.calcite.avatica:avatica&package-manager=maven&previous-version=1.24.0&new-version=1.25.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: David Li --- java/flight/flight-sql-jdbc-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/flight/flight-sql-jdbc-core/pom.xml b/java/flight/flight-sql-jdbc-core/pom.xml index fbab69df3b305..459412e0f8d8b 100644 --- a/java/flight/flight-sql-jdbc-core/pom.xml +++ b/java/flight/flight-sql-jdbc-core/pom.xml @@ -125,7 +125,7 @@ org.apache.calcite.avatica avatica - 1.24.0 + 1.25.0 From 8b2c7e2893fec41c0b6ed51f5e05a7510b427f64 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Jun 2024 16:16:42 +0900 Subject: [PATCH 70/93] MINOR: [Java] Bump com.github.luben:zstd-jni from 1.5.5-11 to 1.5.6-3 in /java (#41521) Bumps [com.github.luben:zstd-jni](https://github.com/luben/zstd-jni) from 1.5.5-11 to 1.5.6-3.
Commits
  • c77a765 v1.5.6-3
  • 2d33a1e Fix missing Native free of Dict: Even when using a by-reference buffer, the d...
  • 1ff8933 Use cross-compile for i386
  • b833326 Adding a getByReferenceBuffer() method.
  • 2a262bf Add new constructor to ZstdDictCompress and ZstdDictDecompress that
  • a516a43 Add back some inspection on MacOS, bump version
  • 2e00ab1 Enable tests on M1 MacOS
  • c76455c Add debugging in MacOS CI
  • ec1ddeb Use the M1 MacOS runner to compile the aarch64 binary
  • fb6a35d Update also checkout and setup-qemu actions
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=com.github.luben:zstd-jni&package-manager=maven&previous-version=1.5.5-11&new-version=1.5.6-3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
> **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: David Li --- java/compression/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/compression/pom.xml b/java/compression/pom.xml index 6ed0be6815ca3..a3d6c0ac558dd 100644 --- a/java/compression/pom.xml +++ b/java/compression/pom.xml @@ -47,7 +47,7 @@ com.github.luben zstd-jni - 1.5.5-11 + 1.5.6-3 From cea7323772beb5be6e5002624be36e6575a36418 Mon Sep 17 00:00:00 2001 From: Vibhatha Lakmal Abeykoon Date: Thu, 6 Jun 2024 13:07:11 +0530 Subject: [PATCH 71/93] GH-41968: [Java] Implement TransferPair functionality for BinaryView (#41980) ### Rationale for this change This PR contains the transferPair functionality for BinaryView vectors. ### What changes are included in this PR? This includes the addition of transferPair functionality in `ViewVarCharBinaryVector` and corresponding test cases. ### Are these changes tested? Yes ### Are there any user-facing changes? No * GitHub Issue: #41968 Authored-by: Vibhatha Abeykoon Signed-off-by: David Li --- .../arrow/vector/ViewVarBinaryVector.java | 45 +- .../arrow/vector/TestSplitAndTransfer.java | 392 +++++---- .../arrow/vector/TestVarCharViewVector.java | 814 ++++++++++-------- 3 files changed, 756 insertions(+), 495 deletions(-) diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ViewVarBinaryVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ViewVarBinaryVector.java index 393df96b2969e..0a043b51067ef 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ViewVarBinaryVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ViewVarBinaryVector.java @@ -205,14 +205,12 @@ public void setSafe(int index, NullableViewVarBinaryHolder holder) { */ @Override public TransferPair getTransferPair(String ref, BufferAllocator allocator) { - // TODO: https://github.com/apache/arrow/issues/40932 - throw new UnsupportedOperationException("Unsupported operation"); + return new TransferImpl(ref, allocator); } @Override public TransferPair getTransferPair(Field field, BufferAllocator allocator) { - // TODO: https://github.com/apache/arrow/issues/40932 - throw new UnsupportedOperationException("Unsupported operation"); + return new TransferImpl(field, allocator); } /** @@ -223,7 +221,42 @@ public TransferPair getTransferPair(Field field, BufferAllocator allocator) { */ @Override public TransferPair makeTransferPair(ValueVector to) { - // TODO: https://github.com/apache/arrow/issues/40932 - throw new UnsupportedOperationException("Unsupported operation"); + return new TransferImpl((ViewVarBinaryVector) to); + } + + private class TransferImpl implements TransferPair { + ViewVarBinaryVector to; + + public TransferImpl(String ref, BufferAllocator allocator) { + to = new ViewVarBinaryVector(ref, field.getFieldType(), allocator); + } + + public TransferImpl(Field field, BufferAllocator allocator) { + to = new ViewVarBinaryVector(field, allocator); + } + + public TransferImpl(ViewVarBinaryVector to) { + this.to = to; + } + + @Override + public ViewVarBinaryVector getTo() { + return to; + } + + @Override + public void transfer() { + transferTo(to); + } + + @Override + public void splitAndTransfer(int startIndex, int length) { + splitAndTransferTo(startIndex, length, to); + } + + @Override + public void copyValueSafe(int fromIndex, int toIndex) { + to.copyFromSafe(fromIndex, toIndex, ViewVarBinaryVector.this); + } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java b/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java index d2c03930ca37a..fece93de9bf14 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestSplitAndTransfer.java @@ -67,7 +67,8 @@ private void populateVarcharVector(final VarCharVector vector, int valueCount, S vector.setValueCount(valueCount); } - private void populateViewVarcharVector(final ViewVarCharVector vector, int valueCount, String[] compareArray) { + private void populateBaseVariableWidthViewVector(final BaseVariableWidthViewVector vector, int valueCount, + String[] compareArray) { for (int i = 0; i < valueCount; i += 3) { final String s = String.format("%010d", i); vector.set(i, s.getBytes(StandardCharsets.UTF_8)); @@ -120,11 +121,16 @@ public void testWithEmptyVector() { transferPair = varCharVector.getTransferPair(allocator); transferPair.splitAndTransfer(0, 0); assertEquals(0, transferPair.getTo().getValueCount()); - // BaseVariableWidthViewVector + // BaseVariableWidthViewVector: ViewVarCharVector ViewVarCharVector viewVarCharVector = new ViewVarCharVector("", allocator); transferPair = viewVarCharVector.getTransferPair(allocator); transferPair.splitAndTransfer(0, 0); assertEquals(0, transferPair.getTo().getValueCount()); + // BaseVariableWidthVector: ViewVarBinaryVector + ViewVarBinaryVector viewVarBinaryVector = new ViewVarBinaryVector("", allocator); + transferPair = viewVarBinaryVector.getTransferPair(allocator); + transferPair.splitAndTransfer(0, 0); + assertEquals(0, transferPair.getTo().getValueCount()); // BaseLargeVariableWidthVector LargeVarCharVector largeVarCharVector = new LargeVarCharVector("", allocator); transferPair = largeVarCharVector.getTransferPair(allocator); @@ -225,36 +231,46 @@ public void test() throws Exception { } } - @Test - public void testView() throws Exception { - try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator)) { - viewVarCharVector.allocateNew(10000, 1000); - - final int valueCount = 500; - final String[] compareArray = new String[valueCount]; + private void testView(BaseVariableWidthViewVector vector) { + vector.allocateNew(10000, 1000); + final int valueCount = 500; + final String[] compareArray = new String[valueCount]; - populateViewVarcharVector(viewVarCharVector, valueCount, compareArray); + populateBaseVariableWidthViewVector(vector, valueCount, compareArray); - final TransferPair tp = viewVarCharVector.getTransferPair(allocator); - final ViewVarCharVector newViewVarCharVector = (ViewVarCharVector) tp.getTo(); - final int[][] startLengths = {{0, 201}, {201, 0}, {201, 200}, {401, 99}}; + final TransferPair tp = vector.getTransferPair(allocator); + final BaseVariableWidthViewVector newVector = (BaseVariableWidthViewVector) tp.getTo();; + final int[][] startLengths = {{0, 201}, {201, 0}, {201, 200}, {401, 99}}; - for (final int[] startLength : startLengths) { - final int start = startLength[0]; - final int length = startLength[1]; - tp.splitAndTransfer(start, length); - for (int i = 0; i < length; i++) { - final boolean expectedSet = ((start + i) % 3) == 0; - if (expectedSet) { - final byte[] expectedValue = compareArray[start + i].getBytes(StandardCharsets.UTF_8); - assertFalse(newViewVarCharVector.isNull(i)); - assertArrayEquals(expectedValue, newViewVarCharVector.get(i)); - } else { - assertTrue(newViewVarCharVector.isNull(i)); - } + for (final int[] startLength : startLengths) { + final int start = startLength[0]; + final int length = startLength[1]; + tp.splitAndTransfer(start, length); + for (int i = 0; i < length; i++) { + final boolean expectedSet = ((start + i) % 3) == 0; + if (expectedSet) { + final byte[] expectedValue = compareArray[start + i].getBytes(StandardCharsets.UTF_8); + assertFalse(newVector.isNull(i)); + assertArrayEquals(expectedValue, newVector.get(i)); + } else { + assertTrue(newVector.isNull(i)); } - newViewVarCharVector.clear(); } + newVector.clear(); + } + } + + @Test + public void testUtf8View() { + try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator)) { + testView(viewVarCharVector); + } + } + + @Test + public void testBinaryView() throws Exception { + try (final ViewVarBinaryVector viewVarBinaryVector = new ViewVarBinaryVector("myvector", allocator)) { + testView(viewVarBinaryVector); } } @@ -282,35 +298,47 @@ public void testMemoryConstrainedTransfer() { } } - @Test - public void testMemoryConstrainedTransferInViews() { - try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator)) { - // Here we have the target vector being transferred with a long string - // hence, the data buffer will be allocated. - // The default data buffer allocation takes - // BaseVariableWidthViewVector.INITIAL_VIEW_VALUE_ALLOCATION * BaseVariableWidthViewVector.ELEMENT_SIZE - // set limit = BaseVariableWidthViewVector.INITIAL_VIEW_VALUE_ALLOCATION * - // BaseVariableWidthViewVector.ELEMENT_SIZE - final int setLimit = BaseVariableWidthViewVector.INITIAL_VIEW_VALUE_ALLOCATION * - BaseVariableWidthViewVector.ELEMENT_SIZE; - allocator.setLimit(setLimit); - - viewVarCharVector.allocateNew(16000, 1000); + private void testMemoryConstrainedTransferInViews(BaseVariableWidthViewVector vector) { + // Here we have the target vector being transferred with a long string + // hence, the data buffer will be allocated. + // The default data buffer allocation takes + // BaseVariableWidthViewVector.INITIAL_VIEW_VALUE_ALLOCATION * BaseVariableWidthViewVector.ELEMENT_SIZE + // set limit = BaseVariableWidthViewVector.INITIAL_VIEW_VALUE_ALLOCATION * + // BaseVariableWidthViewVector.ELEMENT_SIZE + final int setLimit = BaseVariableWidthViewVector.INITIAL_VIEW_VALUE_ALLOCATION * + BaseVariableWidthViewVector.ELEMENT_SIZE; + allocator.setLimit(setLimit); - final int valueCount = 1000; + vector.allocateNew(16000, 1000); - populateViewVarcharVector(viewVarCharVector, valueCount, null); + final int valueCount = 1000; - final TransferPair tp = viewVarCharVector.getTransferPair(allocator); - final ViewVarCharVector newViewVarCharVector = (ViewVarCharVector) tp.getTo(); - final int[][] startLengths = {{0, 700}, {700, 299}}; + populateBaseVariableWidthViewVector(vector, valueCount, null); - for (final int[] startLength : startLengths) { - final int start = startLength[0]; - final int length = startLength[1]; - tp.splitAndTransfer(start, length); - newViewVarCharVector.clear(); - } + final TransferPair tp = vector.getTransferPair(allocator); + final BaseVariableWidthViewVector newVector = (BaseVariableWidthViewVector) tp.getTo(); + + final int[][] startLengths = {{0, 700}, {700, 299}}; + + for (final int[] startLength : startLengths) { + final int start = startLength[0]; + final int length = startLength[1]; + tp.splitAndTransfer(start, length); + newVector.clear(); + } + } + + @Test + public void testMemoryConstrainedTransferInUtf8Views() { + try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator)) { + testMemoryConstrainedTransferInViews(viewVarCharVector); + } + } + + @Test + public void testMemoryConstrainedTransferInBinaryViews() { + try (final ViewVarBinaryVector viewVarBinaryVector = new ViewVarBinaryVector("myvector", allocator)) { + testMemoryConstrainedTransferInViews(viewVarBinaryVector); } } @@ -345,34 +373,45 @@ public void testTransfer() { } } - @Test - public void testTransferInViews() { - try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator)) { - viewVarCharVector.allocateNew(16000, 1000); + private void testTransferInViews(BaseVariableWidthViewVector vector) { + vector.allocateNew(16000, 1000); - final int valueCount = 500; - final String[] compareArray = new String[valueCount]; - populateViewVarcharVector(viewVarCharVector, valueCount, compareArray); + final int valueCount = 500; + final String[] compareArray = new String[valueCount]; + populateBaseVariableWidthViewVector(vector, valueCount, compareArray); - final TransferPair tp = viewVarCharVector.getTransferPair(allocator); - final ViewVarCharVector newViewVarCharVector = (ViewVarCharVector) tp.getTo(); - tp.transfer(); + final TransferPair tp = vector.getTransferPair(allocator); + final BaseVariableWidthViewVector newVector = (BaseVariableWidthViewVector) tp.getTo(); + tp.transfer(); - assertEquals(0, viewVarCharVector.valueCount); - assertEquals(valueCount, newViewVarCharVector.valueCount); + assertEquals(0, vector.valueCount); + assertEquals(valueCount, newVector.valueCount); - for (int i = 0; i < valueCount; i++) { - final boolean expectedSet = (i % 3) == 0; - if (expectedSet) { - final byte[] expectedValue = compareArray[i].getBytes(StandardCharsets.UTF_8); - assertFalse(newViewVarCharVector.isNull(i)); - assertArrayEquals(expectedValue, newViewVarCharVector.get(i)); - } else { - assertTrue(newViewVarCharVector.isNull(i)); - } + for (int i = 0; i < valueCount; i++) { + final boolean expectedSet = (i % 3) == 0; + if (expectedSet) { + final byte[] expectedValue = compareArray[i].getBytes(StandardCharsets.UTF_8); + assertFalse(newVector.isNull(i)); + assertArrayEquals(expectedValue, newVector.get(i)); + } else { + assertTrue(newVector.isNull(i)); } + } + + newVector.clear(); + } - newViewVarCharVector.clear(); + @Test + public void testTransferInUtf8Views() { + try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator)) { + testTransferInViews(viewVarCharVector); + } + } + + @Test + public void testTransferInBinaryViews() { + try (final ViewVarBinaryVector viewVarBinaryVector = new ViewVarBinaryVector("myvector", allocator)) { + testTransferInViews(viewVarBinaryVector); } } @@ -424,21 +463,31 @@ public void testSplitAndTransferNon() { } } - @Test - public void testSplitAndTransferNonInViews() { - try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator)) { + private void testSplitAndTransferNonInViews(BaseVariableWidthViewVector vector) { + vector.allocateNew(16000, 1000); + final int valueCount = 500; + populateBaseVariableWidthViewVector(vector, valueCount, null); - viewVarCharVector.allocateNew(16000, 1000); - final int valueCount = 500; - populateViewVarcharVector(viewVarCharVector, valueCount, null); + final TransferPair tp = vector.getTransferPair(allocator); + BaseVariableWidthViewVector newVector = (BaseVariableWidthViewVector) tp.getTo(); - final TransferPair tp = viewVarCharVector.getTransferPair(allocator); - ViewVarCharVector newViewVarCharVector = (ViewVarCharVector) tp.getTo(); + tp.splitAndTransfer(0, 0); + assertEquals(0, newVector.getValueCount()); - tp.splitAndTransfer(0, 0); - assertEquals(0, newViewVarCharVector.getValueCount()); + newVector.clear(); + } - newViewVarCharVector.clear(); + @Test + public void testSplitAndTransferNonInUtf8Views() { + try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator)) { + testSplitAndTransferNonInViews(viewVarCharVector); + } + } + + @Test + public void testSplitAndTransferNonInBinaryViews() { + try (final ViewVarBinaryVector viewVarBinaryVector = new ViewVarBinaryVector("myvector", allocator)) { + testSplitAndTransferNonInViews(viewVarBinaryVector); } } @@ -460,21 +509,31 @@ public void testSplitAndTransferAll() { } } - @Test - public void testSplitAndTransferAllInViews() { - try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator)) { + private void testSplitAndTransferAllInViews(BaseVariableWidthViewVector vector) { + vector.allocateNew(16000, 1000); + final int valueCount = 500; + populateBaseVariableWidthViewVector(vector, valueCount, null); - viewVarCharVector.allocateNew(16000, 1000); - final int valueCount = 500; - populateViewVarcharVector(viewVarCharVector, valueCount, null); + final TransferPair tp = vector.getTransferPair(allocator); + BaseVariableWidthViewVector newViewVarCharVector = (BaseVariableWidthViewVector) tp.getTo(); - final TransferPair tp = viewVarCharVector.getTransferPair(allocator); - ViewVarCharVector newViewVarCharVector = (ViewVarCharVector) tp.getTo(); + tp.splitAndTransfer(0, valueCount); + assertEquals(valueCount, newViewVarCharVector.getValueCount()); - tp.splitAndTransfer(0, valueCount); - assertEquals(valueCount, newViewVarCharVector.getValueCount()); + newViewVarCharVector.clear(); + } - newViewVarCharVector.clear(); + @Test + public void testSplitAndTransferAllInUtf8Views() { + try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator)) { + testSplitAndTransferAllInViews(viewVarCharVector); + } + } + + @Test + public void testSplitAndTransferAllInBinaryViews() { + try (final ViewVarBinaryVector viewVarBinaryVector = new ViewVarBinaryVector("myvector", allocator)) { + testSplitAndTransferAllInViews(viewVarBinaryVector); } } @@ -499,24 +558,35 @@ public void testInvalidStartIndex() { } } - @Test - public void testInvalidStartIndexInViews() { - try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator); - final ViewVarCharVector newViewVarCharVector = new ViewVarCharVector("newvector", allocator)) { + private void testInvalidStartIndexInViews(BaseVariableWidthViewVector vector, BaseVariableWidthViewVector newVector) { + vector.allocateNew(16000, 1000); + final int valueCount = 500; + populateBaseVariableWidthViewVector(vector, valueCount, null); - viewVarCharVector.allocateNew(16000, 1000); - final int valueCount = 500; - populateViewVarcharVector(viewVarCharVector, valueCount, null); + final TransferPair tp = vector.makeTransferPair(newVector); - final TransferPair tp = viewVarCharVector.makeTransferPair(newViewVarCharVector); + IllegalArgumentException e = assertThrows( + IllegalArgumentException.class, + () -> tp.splitAndTransfer(valueCount, 10)); - IllegalArgumentException e = assertThrows( - IllegalArgumentException.class, - () -> tp.splitAndTransfer(valueCount, 10)); + assertEquals("Invalid parameters startIndex: 500, length: 10 for valueCount: 500", e.getMessage()); - assertEquals("Invalid parameters startIndex: 500, length: 10 for valueCount: 500", e.getMessage()); + newVector.clear(); + } - newViewVarCharVector.clear(); + @Test + public void testInvalidStartIndexInUtf8Views() { + try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator); + final ViewVarCharVector newViewVarCharVector = new ViewVarCharVector("newvector", allocator)) { + testInvalidStartIndexInViews(viewVarCharVector, newViewVarCharVector); + } + } + + @Test + public void testInvalidStartIndexInBinaryViews() { + try (final ViewVarBinaryVector viewVarBinaryVector = new ViewVarBinaryVector("myvector", allocator); + final ViewVarBinaryVector newViewVarBinaryVector = new ViewVarBinaryVector("newvector", allocator)) { + testInvalidStartIndexInViews(viewVarBinaryVector, newViewVarBinaryVector); } } @@ -541,24 +611,35 @@ public void testInvalidLength() { } } - @Test - public void testInvalidLengthInViews() { - try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator); - final ViewVarCharVector newViewVarCharVector = new ViewVarCharVector("newvector", allocator)) { + private void testInvalidLengthInViews(BaseVariableWidthViewVector vector, BaseVariableWidthViewVector newVector) { + vector.allocateNew(16000, 1000); + final int valueCount = 500; + populateBaseVariableWidthViewVector(vector, valueCount, null); - viewVarCharVector.allocateNew(16000, 1000); - final int valueCount = 500; - populateViewVarcharVector(viewVarCharVector, valueCount, null); + final TransferPair tp = vector.makeTransferPair(newVector); - final TransferPair tp = viewVarCharVector.makeTransferPair(newViewVarCharVector); + IllegalArgumentException e = assertThrows( + IllegalArgumentException.class, + () -> tp.splitAndTransfer(0, valueCount * 2)); - IllegalArgumentException e = assertThrows( - IllegalArgumentException.class, - () -> tp.splitAndTransfer(0, valueCount * 2)); + assertEquals("Invalid parameters startIndex: 0, length: 1000 for valueCount: 500", e.getMessage()); - assertEquals("Invalid parameters startIndex: 0, length: 1000 for valueCount: 500", e.getMessage()); + newVector.clear(); + } - newViewVarCharVector.clear(); + @Test + public void testInvalidLengthInUtf8Views() { + try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator); + final ViewVarCharVector newViewVarCharVector = new ViewVarCharVector("newvector", allocator)) { + testInvalidLengthInViews(viewVarCharVector, newViewVarCharVector); + } + } + + @Test + public void testInvalidLengthInBinaryViews() { + try (final ViewVarBinaryVector viewVarBinaryVector = new ViewVarBinaryVector("myvector", allocator); + final ViewVarBinaryVector newViewVarBinaryVector = new ViewVarBinaryVector("newvector", allocator)) { + testInvalidLengthInViews(viewVarBinaryVector, newViewVarBinaryVector); } } @@ -580,21 +661,33 @@ public void testZeroStartIndexAndLength() { } } - @Test - public void testZeroStartIndexAndLengthInViews() { - try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator); - final ViewVarCharVector newViewVarCharVector = new ViewVarCharVector("newvector", allocator)) { + private void testZeroStartIndexAndLengthInViews(BaseVariableWidthViewVector vector, + BaseVariableWidthViewVector newVector) { + vector.allocateNew(0, 0); + final int valueCount = 0; + populateBaseVariableWidthViewVector(vector, valueCount, null); - viewVarCharVector.allocateNew(0, 0); - final int valueCount = 0; - populateViewVarcharVector(viewVarCharVector, valueCount, null); + final TransferPair tp = vector.makeTransferPair(newVector); - final TransferPair tp = viewVarCharVector.makeTransferPair(newViewVarCharVector); + tp.splitAndTransfer(0, 0); + assertEquals(valueCount, newVector.getValueCount()); - tp.splitAndTransfer(0, 0); - assertEquals(valueCount, newViewVarCharVector.getValueCount()); + newVector.clear(); + } - newViewVarCharVector.clear(); + @Test + public void testZeroStartIndexAndLengthInUtf8Views() { + try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator); + final ViewVarCharVector newViewVarCharVector = new ViewVarCharVector("newvector", allocator)) { + testZeroStartIndexAndLengthInViews(viewVarCharVector, newViewVarCharVector); + } + } + + @Test + public void testZeroStartIndexAndLengthInBinaryViews() { + try (final ViewVarBinaryVector viewVarBinaryVector = new ViewVarBinaryVector("myvector", allocator); + final ViewVarBinaryVector newViewVarBinaryVector = new ViewVarBinaryVector("newvector", allocator)) { + testZeroStartIndexAndLengthInViews(viewVarBinaryVector, newViewVarBinaryVector); } } @@ -616,21 +709,32 @@ public void testZeroLength() { } } - @Test - public void testZeroLengthInViews() { - try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator); - final ViewVarCharVector newViewVarCharVector = new ViewVarCharVector("newvector", allocator)) { + private void testZeroLengthInViews(BaseVariableWidthViewVector vector, BaseVariableWidthViewVector newVector) { + vector.allocateNew(16000, 1000); + final int valueCount = 500; + populateBaseVariableWidthViewVector(vector, valueCount, null); - viewVarCharVector.allocateNew(16000, 1000); - final int valueCount = 500; - populateViewVarcharVector(viewVarCharVector, valueCount, null); + final TransferPair tp = vector.makeTransferPair(newVector); - final TransferPair tp = viewVarCharVector.makeTransferPair(newViewVarCharVector); + tp.splitAndTransfer(500, 0); + assertEquals(0, newVector.getValueCount()); - tp.splitAndTransfer(500, 0); - assertEquals(0, newViewVarCharVector.getValueCount()); + newVector.clear(); + } - newViewVarCharVector.clear(); + @Test + public void testZeroLengthInUtf8Views() { + try (final ViewVarCharVector viewVarCharVector = new ViewVarCharVector("myvector", allocator); + final ViewVarCharVector newViewVarCharVector = new ViewVarCharVector("newvector", allocator)) { + testZeroLengthInViews(viewVarCharVector, newViewVarCharVector); + } + } + + @Test + public void testZeroLengthInBinaryViews() { + try (final ViewVarBinaryVector viewVarBinaryVector = new ViewVarBinaryVector("myvector", allocator); + final ViewVarBinaryVector newViewVarBinaryVector = new ViewVarBinaryVector("newvector", allocator)) { + testZeroLengthInViews(viewVarBinaryVector, newViewVarBinaryVector); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharViewVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharViewVector.java index 817941ecb46d6..ebf9b58da7b40 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharViewVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharViewVector.java @@ -37,7 +37,9 @@ import java.util.List; import java.util.Objects; import java.util.Random; +import java.util.function.BiConsumer; import java.util.function.Function; +import java.util.stream.IntStream; import java.util.stream.Stream; import org.apache.arrow.memory.ArrowBuf; @@ -86,9 +88,12 @@ public class TestVarCharViewVector { private BufferAllocator allocator; + private Random random; + @BeforeEach public void prepare() { allocator = new RootAllocator(Integer.MAX_VALUE); + random = new Random(); } @AfterEach @@ -1717,6 +1722,44 @@ public void testCopyFromSafeWithNulls(Function validateVector = (targetVector, expectedData) -> { + IntStream.range(startIndex, length).forEach(i -> + assertArrayEquals(expectedData[i], targetVector.get(i - startIndex))); + }; + try (final ViewVarCharVector targetVector = newViewVarCharVector("split-target", allocator)) { try (final ViewVarCharVector sourceVector = newViewVarCharVector(EMPTY_SCHEMA_PATH, allocator)) { - sourceVector.allocateNew(1024 * 10, 1024); - - sourceVector.set(0, STR4); - sourceVector.set(1, STR5); - sourceVector.set(2, STR6); - sourceVector.setValueCount(3); - - final long allocatedMem = allocator.getAllocatedMemory(); - final int validityRefCnt = sourceVector.getValidityBuffer().refCnt(); - final int dataRefCnt = sourceVector.getDataBuffer().refCnt(); - - sourceVector.splitAndTransferTo(0, 2, targetVector); - // we allocate view and data buffers for the target vector - assertTrue(allocatedMem < allocator.getAllocatedMemory()); - - // The validity buffer is sliced from the same buffer.See BaseFixedWidthViewVector#allocateBytes. - // Therefore, the refcnt of the validity buffer is increased once since the startIndex is 0. - assertEquals(validityRefCnt + 1, sourceVector.getValidityBuffer().refCnt()); - // since the new view buffer is allocated, the refcnt is the same as the source vector. - assertEquals(dataRefCnt, sourceVector.getDataBuffer().refCnt()); + testSplitAndTransferOnSlicedBufferHelper(targetVector, sourceVector, + startIndex, length, data); } - assertArrayEquals(STR4, targetVector.get(0)); - assertArrayEquals(STR5, targetVector.get(1)); + validateVector.accept(targetVector, data); + } + + final byte [][] binaryData = generateBinaryDataArray(3, 10); + + try (final ViewVarBinaryVector targetVector = newViewVarBinaryVector("split-target", allocator)) { + try (final ViewVarBinaryVector sourceVector = newViewVarBinaryVector(EMPTY_SCHEMA_PATH, allocator)) { + testSplitAndTransferOnSlicedBufferHelper(targetVector, sourceVector, + startIndex, length, binaryData); + } + validateVector.accept(targetVector, binaryData); } } @@ -1760,36 +1803,58 @@ public void testSplitAndTransfer1() { * With a long string included. */ @Test - public void testSplitAndTransfer2() { + public void testSplitAndTransferWithLongStringsOnSlicedBuffer() { + final byte [][] data = new byte[][]{STR2, STR5, STR6}; + final int startIndex = 0; + final int length = 2; + + BiConsumer validateVector = (targetVector, expectedData) -> { + IntStream.range(startIndex, length).forEach(i -> + assertArrayEquals(expectedData[i], targetVector.get(i - startIndex))); + }; + try (final ViewVarCharVector targetVector = newViewVarCharVector("split-target", allocator)) { try (final ViewVarCharVector sourceVector = newViewVarCharVector(EMPTY_SCHEMA_PATH, allocator)) { - sourceVector.allocateNew(1024 * 10, 1024); - - sourceVector.set(0, STR2); - sourceVector.set(1, STR5); - sourceVector.set(2, STR6); - sourceVector.setValueCount(3); - - final long allocatedMem = allocator.getAllocatedMemory(); - final int validityRefCnt = sourceVector.getValidityBuffer().refCnt(); - final int dataRefCnt = sourceVector.getDataBuffer().refCnt(); - - sourceVector.splitAndTransferTo(0, 2, targetVector); - // we allocate view and data buffers for the target vector - assertTrue(allocatedMem < allocator.getAllocatedMemory()); - - // The validity buffer is sliced from the same buffer.See BaseFixedWidthViewVector#allocateBytes. - // Therefore, the refcnt of the validity buffer is increased once since the startIndex is 0. - assertEquals(validityRefCnt + 1, sourceVector.getValidityBuffer().refCnt()); - // since the new view buffer is allocated, the refcnt is the same as the source vector. - assertEquals(dataRefCnt, sourceVector.getDataBuffer().refCnt()); - - assertArrayEquals(STR2, sourceVector.get(0)); - assertArrayEquals(STR5, sourceVector.get(1)); - assertArrayEquals(STR6, sourceVector.get(2)); + testSplitAndTransferOnSlicedBufferHelper(targetVector, sourceVector, + startIndex, length, data); } - assertArrayEquals(STR2, targetVector.get(0)); - assertArrayEquals(STR5, targetVector.get(1)); + validateVector.accept(targetVector, data); + } + + final byte [][] binaryData = generateBinaryDataArray(3, 18); + try (final ViewVarBinaryVector targetVector = newViewVarBinaryVector("split-target", allocator)) { + try (final ViewVarBinaryVector sourceVector = newViewVarBinaryVector(EMPTY_SCHEMA_PATH, allocator)) { + testSplitAndTransferOnSlicedBufferHelper(targetVector, sourceVector, + startIndex, length, binaryData); + } + validateVector.accept(targetVector, binaryData); + } + } + + private void testSplitAndTransferOnSlicedVectorHelper(BaseVariableWidthViewVector sourceVector, + BaseVariableWidthViewVector targetVector, int startIndex, int length, byte[][] data) { + sourceVector.allocateNew(1024 * 10, 1024); + + for (int i = 0; i < data.length; i++) { + sourceVector.set(i, data[i]); + } + sourceVector.setValueCount(data.length); + + final long allocatedMem = allocator.getAllocatedMemory(); + final int validityRefCnt = sourceVector.getValidityBuffer().refCnt(); + final int dataRefCnt = sourceVector.getDataBuffer().refCnt(); + + sourceVector.splitAndTransferTo(startIndex, length, targetVector); + // we allocate view and data buffers for the target vector + assertTrue(allocatedMem < allocator.getAllocatedMemory()); + // The validity buffer is sliced from the same buffer.See BaseFixedWidthViewVector#allocateBytes. + // Therefore, the refcnt of the validity buffer is increased once since the startIndex is 0. + assertEquals(validityRefCnt + 1, sourceVector.getValidityBuffer().refCnt()); + // since the new view buffer is allocated, the refcnt is the same as the source vector. + assertEquals(dataRefCnt, sourceVector.getDataBuffer().refCnt()); + + for (int i = startIndex; i < length ; i++) { + assertArrayEquals(data[i], targetVector.get(i - startIndex)); } } @@ -1800,35 +1865,31 @@ public void testSplitAndTransfer2() { * With short strings. */ @Test - public void testSplitAndTransfer3() { + public void testSplitAndTransferWithShortStringsOnSlicedVector() { + byte [][] data = new byte[][]{STR4, STR5, STR6}; + final int startIndex = 0; + final int length = 2; + + BiConsumer validateVector = (sourceVector, expectedData) -> { + IntStream.range(startIndex, length).forEach(i -> + assertArrayEquals(expectedData[i], sourceVector.get(i))); + }; + try (final ViewVarCharVector sourceVector = newViewVarCharVector(EMPTY_SCHEMA_PATH, allocator)) { try (final ViewVarCharVector targetVector = newViewVarCharVector("split-target", allocator)) { - sourceVector.allocateNew(1024 * 10, 1024); - - sourceVector.set(0, STR4); - sourceVector.set(1, STR5); - sourceVector.set(2, STR6); - sourceVector.setValueCount(3); - - final long allocatedMem = allocator.getAllocatedMemory(); - final int validityRefCnt = sourceVector.getValidityBuffer().refCnt(); - final int dataRefCnt = sourceVector.getDataBuffer().refCnt(); - - sourceVector.splitAndTransferTo(0, 2, targetVector); - // we allocate view and data buffers for the target vector - assertTrue(allocatedMem < allocator.getAllocatedMemory()); - // The validity buffer is sliced from the same buffer.See BaseFixedWidthViewVector#allocateBytes. - // Therefore, the refcnt of the validity buffer is increased once since the startIndex is 0. - assertEquals(validityRefCnt + 1, sourceVector.getValidityBuffer().refCnt()); - // since the new view buffer is allocated, the refcnt is the same as the source vector. - assertEquals(dataRefCnt, sourceVector.getDataBuffer().refCnt()); - - assertArrayEquals(STR4, targetVector.get(0)); - assertArrayEquals(STR5, targetVector.get(1)); + testSplitAndTransferOnSlicedVectorHelper(sourceVector, targetVector, + startIndex, length, data); + } + validateVector.accept(sourceVector, data); + } + + byte [][] binaryData = generateBinaryDataArray(3, 10); + try (final ViewVarBinaryVector sourceVector = newViewVarBinaryVector(EMPTY_SCHEMA_PATH, allocator)) { + try (final ViewVarBinaryVector targetVector = newViewVarBinaryVector("split-target", allocator)) { + testSplitAndTransferOnSlicedVectorHelper(sourceVector, targetVector, + startIndex, length, binaryData); } - assertArrayEquals(STR4, sourceVector.get(0)); - assertArrayEquals(STR5, sourceVector.get(1)); - assertArrayEquals(STR6, sourceVector.get(2)); + validateVector.accept(sourceVector, binaryData); } } @@ -1839,35 +1900,77 @@ public void testSplitAndTransfer3() { * With a long string included. */ @Test - public void testSplitAndTransfer4() { + public void testSplitAndTransferWithLongStringsOnSlicedVector() { + final byte [][] data = new byte[][]{STR2, STR5, STR6}; + final int startIndex = 0; + final int length = 2; + + BiConsumer validateVector = (sourceVector, expectedData) -> { + IntStream.range(startIndex, length).forEach(i -> + assertArrayEquals(expectedData[i], sourceVector.get(i))); + }; + try (final ViewVarCharVector sourceVector = newViewVarCharVector(EMPTY_SCHEMA_PATH, allocator)) { try (final ViewVarCharVector targetVector = newViewVarCharVector("split-target", allocator)) { - sourceVector.allocateNew(1024 * 10, 1024); - - sourceVector.set(0, STR2); - sourceVector.set(1, STR5); - sourceVector.set(2, STR6); - sourceVector.setValueCount(3); - - final long allocatedMem = allocator.getAllocatedMemory(); - final int validityRefCnt = sourceVector.getValidityBuffer().refCnt(); - final int dataRefCnt = sourceVector.getDataBuffer().refCnt(); - - sourceVector.splitAndTransferTo(0, 2, targetVector); - // we allocate view and data buffers for the target vector - assertTrue(allocatedMem < allocator.getAllocatedMemory()); - // The validity buffer is sliced from the same buffer.See BaseFixedWidthViewVector#allocateBytes. - // Therefore, the refcnt of the validity buffer is increased once since the startIndex is 0. - assertEquals(validityRefCnt + 1, sourceVector.getValidityBuffer().refCnt()); - // since the new view buffer is allocated, the refcnt is the same as the source vector. - assertEquals(dataRefCnt, sourceVector.getDataBuffer().refCnt()); - - assertArrayEquals(STR2, targetVector.get(0)); - assertArrayEquals(STR5, targetVector.get(1)); + testSplitAndTransferOnSlicedVectorHelper(sourceVector, targetVector, + startIndex, length, data); + } + validateVector.accept(sourceVector, data); + } + + final byte [][] binaryData = generateBinaryDataArray(3, 20); + try (final ViewVarBinaryVector sourceVector = newViewVarBinaryVector(EMPTY_SCHEMA_PATH, allocator)) { + try (final ViewVarBinaryVector targetVector = newViewVarBinaryVector("split-target", allocator)) { + testSplitAndTransferOnSlicedVectorHelper(sourceVector, targetVector, + startIndex, length, binaryData); + } + validateVector.accept(sourceVector, binaryData); + } + } + + private void testSplitAndTransferOnValiditySplitHelper( + BaseVariableWidthViewVector targetVector, BaseVariableWidthViewVector sourceVector, + int startIndex, int length, byte[][] data) { + sourceVector.allocateNew(1024 * 10, 1024); + + sourceVector.set(0, new byte[0]); + sourceVector.setNull(1); + for (int i = 0; i < data.length; i++) { + if (data[i] == null) { + sourceVector.setNull(i); + } else { + sourceVector.set(i, data[i]); + } + } + sourceVector.setValueCount(data.length); + + final long allocatedMem = allocator.getAllocatedMemory(); + final int validityRefCnt = sourceVector.getValidityBuffer().refCnt(); + final int dataRefCnt = sourceVector.getDataBuffer().refCnt(); + + sourceVector.splitAndTransferTo(startIndex, length, targetVector); + // the allocation only consists in the size needed for the validity buffer + final long validitySize = + DefaultRoundingPolicy.DEFAULT_ROUNDING_POLICY.getRoundedSize( + BaseValueVector.getValidityBufferSizeFromCount(2)); + // we allocate view and data buffers for the target vector + assertTrue(allocatedMem + validitySize < allocator.getAllocatedMemory()); + // The validity is sliced from the same buffer.See BaseFixedWidthViewVector#allocateBytes. + // Since values up to the startIndex are empty/null validity refcnt should not change. + assertEquals(validityRefCnt, sourceVector.getValidityBuffer().refCnt()); + // since the new view buffer is allocated, the refcnt is the same as the source vector. + assertEquals(dataRefCnt, sourceVector.getDataBuffer().refCnt()); + + for (int i = startIndex; i < startIndex + length; i++) { + assertArrayEquals(data[i], targetVector.get(i - startIndex)); + } + + for (int i = 0; i < data.length; i++) { + if (data[i] == null) { + assertTrue(sourceVector.isNull(i)); + } else { + assertArrayEquals(data[i], sourceVector.get(i)); } - assertArrayEquals(STR2, sourceVector.get(0)); - assertArrayEquals(STR5, sourceVector.get(1)); - assertArrayEquals(STR6, sourceVector.get(2)); } } @@ -1878,43 +1981,24 @@ public void testSplitAndTransfer4() { * With short strings. */ @Test - public void testSplitAndTransfer5() { + public void testSplitAndTransferWithShortStringsOnValiditySplit() { + final byte [][] data = new byte[][]{new byte[0], null, STR4, STR5, STR6}; + final int startIndex = 2; + final int length = 2; + try (final ViewVarCharVector targetVector = newViewVarCharVector("split-target", allocator); final ViewVarCharVector sourceVector = newViewVarCharVector(EMPTY_SCHEMA_PATH, allocator)) { - sourceVector.allocateNew(1024 * 10, 1024); - - sourceVector.set(0, new byte[0]); - sourceVector.setNull(1); - sourceVector.set(2, STR4); - sourceVector.set(3, STR5); - sourceVector.set(4, STR6); - sourceVector.setValueCount(5); - - final long allocatedMem = allocator.getAllocatedMemory(); - final int validityRefCnt = sourceVector.getValidityBuffer().refCnt(); - final int dataRefCnt = sourceVector.getDataBuffer().refCnt(); - - sourceVector.splitAndTransferTo(2, 2, targetVector); - // the allocation only consists in the size needed for the validity buffer - final long validitySize = - DefaultRoundingPolicy.DEFAULT_ROUNDING_POLICY.getRoundedSize( - BaseValueVector.getValidityBufferSizeFromCount(2)); - // we allocate view and data buffers for the target vector - assertTrue(allocatedMem + validitySize < allocator.getAllocatedMemory()); - // The validity is sliced from the same buffer.See BaseFixedWidthViewVector#allocateBytes. - // Since values up to the startIndex are empty/null validity refcnt should not change. - assertEquals(validityRefCnt, sourceVector.getValidityBuffer().refCnt()); - // since the new view buffer is allocated, the refcnt is the same as the source vector. - assertEquals(dataRefCnt, sourceVector.getDataBuffer().refCnt()); - - assertArrayEquals(STR4, targetVector.get(0)); - assertArrayEquals(STR5, targetVector.get(1)); - - assertArrayEquals(new byte[0], sourceVector.get(0)); - assertTrue(sourceVector.isNull(1)); - assertArrayEquals(STR4, sourceVector.get(2)); - assertArrayEquals(STR5, sourceVector.get(3)); - assertArrayEquals(STR6, sourceVector.get(4)); + testSplitAndTransferOnValiditySplitHelper(targetVector, sourceVector, + startIndex, length, data); + } + + final byte [][] binaryData = generateBinaryDataArray(5, 10); + binaryData[0] = new byte[0]; + binaryData[1] = null; + try (final ViewVarBinaryVector targetVector = newViewVarBinaryVector("split-target", allocator); + final ViewVarBinaryVector sourceVector = newViewVarBinaryVector(EMPTY_SCHEMA_PATH, allocator)) { + testSplitAndTransferOnValiditySplitHelper(targetVector, sourceVector, + startIndex, length, binaryData); } } @@ -1925,44 +2009,59 @@ public void testSplitAndTransfer5() { * With long strings. */ @Test - public void testSplitAndTransfer6() { + public void testSplitAndTransferWithLongStringsOnValiditySplit() { + final byte [][] data = new byte[][]{new byte[0], null, STR1, STR2, STR3}; + final int startIndex = 2; + final int length = 2; + try (final ViewVarCharVector targetVector = newViewVarCharVector("split-target", allocator); final ViewVarCharVector sourceVector = newViewVarCharVector(EMPTY_SCHEMA_PATH, allocator)) { - sourceVector.allocateNew(1024 * 10, 1024); - - sourceVector.set(0, new byte[0]); - sourceVector.setNull(1); - sourceVector.set(2, STR1); - sourceVector.set(3, STR2); - sourceVector.set(4, STR3); - sourceVector.setValueCount(5); - - final long allocatedMem = allocator.getAllocatedMemory(); - final int validityRefCnt = sourceVector.getValidityBuffer().refCnt(); - final int dataRefCnt = sourceVector.getDataBuffer().refCnt(); - - sourceVector.splitAndTransferTo(2, 2, targetVector); - // the allocation consists in the size needed for the validity buffer and the long string - // allocation - final long validitySize = - DefaultRoundingPolicy.DEFAULT_ROUNDING_POLICY.getRoundedSize( - BaseValueVector.getValidityBufferSizeFromCount(2)); - // we allocate view and data buffers for the target vector - assertTrue(allocatedMem + validitySize < allocator.getAllocatedMemory()); - // The validity is sliced from the same buffer.See BaseFixedWidthViewVector#allocateBytes. - // Since values up to the startIndex are empty/null validity refcnt should not change. - assertEquals(validityRefCnt, sourceVector.getValidityBuffer().refCnt()); - // since the new view buffer is allocated, the refcnt is the same as the source vector. - assertEquals(dataRefCnt, sourceVector.getDataBuffer().refCnt()); - - assertArrayEquals(STR1, targetVector.get(0)); - assertArrayEquals(STR2, targetVector.get(1)); - - assertArrayEquals(new byte[0], sourceVector.get(0)); - assertTrue(sourceVector.isNull(1)); - assertArrayEquals(STR1, sourceVector.get(2)); - assertArrayEquals(STR2, sourceVector.get(3)); - assertArrayEquals(STR3, sourceVector.get(4)); + testSplitAndTransferOnValiditySplitHelper(targetVector, sourceVector, + startIndex, length, data); + } + + final byte [][] binaryData = generateBinaryDataArray(5, 18); + binaryData[0] = new byte[0]; + binaryData[1] = null; + + try (final ViewVarBinaryVector targetVector = newViewVarBinaryVector("split-target", allocator); + final ViewVarBinaryVector sourceVector = newViewVarBinaryVector(EMPTY_SCHEMA_PATH, allocator)) { + testSplitAndTransferOnValiditySplitHelper(targetVector, sourceVector, + startIndex, length, data); + } + } + + private void testSplitAndTransferOnAllocatorToAllocator(BaseVariableWidthViewVector targetVector, + BaseVariableWidthViewVector sourceVector, int startIndex, int length, byte[][] data) { + sourceVector.allocateNew(50, data.length); + + for (int i = 0; i < data.length; i++) { + sourceVector.set(i, data[i]); + } + sourceVector.setValueCount(data.length); + + final long allocatedMem = allocator.getAllocatedMemory(); + final int validityRefCnt = sourceVector.getValidityBuffer().refCnt(); + final int dataRefCnt = sourceVector.getDataBuffer().refCnt(); + + sourceVector.splitAndTransferTo(startIndex, length, targetVector); + + if (sourceVector.getDataBuffers().isEmpty()) { + // no extra allocation as strings are all inline + assertEquals(allocatedMem, allocator.getAllocatedMemory()); + } else { + // extra allocation as some strings are not inline + assertTrue(allocatedMem < allocator.getAllocatedMemory()); + } + + // the refcnts of each buffer for this test should be the same as what + // the source allocator ended up with. + assertEquals(validityRefCnt, sourceVector.getValidityBuffer().refCnt()); + // since the new view buffer is allocated, the refcnt is the same as the source vector. + assertEquals(dataRefCnt, sourceVector.getDataBuffer().refCnt()); + + for (int i = 0; i < data.length; i++) { + assertArrayEquals(data[i], sourceVector.get(i)); } } @@ -1973,39 +2072,36 @@ public void testSplitAndTransfer6() { * With short strings. */ @Test - public void testSplitAndTransfer7() { + public void testSplitAndTransferWithShortStringsOnAllocatorToAllocator() { final int maxAllocation = 512; + final byte [][] data = new byte[][]{STR4, STR5, STR6}; + final int startIndex = 0; + final int length = 2; + + BiConsumer validateVector = (targetVector, expectedData) -> { + IntStream.range(startIndex, length).forEach(i -> + assertArrayEquals(expectedData[i], targetVector.get(i - startIndex))); + }; + try (final BufferAllocator targetAllocator = allocator.newChildAllocator("target-alloc", 256, maxAllocation); final ViewVarCharVector targetVector = newViewVarCharVector("split-target", targetAllocator)) { try (final BufferAllocator sourceAllocator = allocator.newChildAllocator("source-alloc", 256, maxAllocation); final ViewVarCharVector sourceVector = newViewVarCharVector(EMPTY_SCHEMA_PATH, sourceAllocator)) { - sourceVector.allocateNew(50, 3); - - sourceVector.set(0, STR4); - sourceVector.set(1, STR5); - sourceVector.set(2, STR6); - sourceVector.setValueCount(3); - - final long allocatedMem = allocator.getAllocatedMemory(); - final int validityRefCnt = sourceVector.getValidityBuffer().refCnt(); - final int dataRefCnt = sourceVector.getDataBuffer().refCnt(); - - sourceVector.splitAndTransferTo(0, 2, targetVector); - // no extra allocation as strings are all inline - assertEquals(allocatedMem, allocator.getAllocatedMemory()); - - // the refcnts of each buffer for this test should be the same as what - // the source allocator ended up with. - assertEquals(validityRefCnt, sourceVector.getValidityBuffer().refCnt()); - // since the new view buffer is allocated, the refcnt is the same as the source vector. - assertEquals(dataRefCnt, sourceVector.getDataBuffer().refCnt()); - - assertArrayEquals(STR4, sourceVector.get(0)); - assertArrayEquals(STR5, sourceVector.get(1)); - assertArrayEquals(STR6, sourceVector.get(2)); + testSplitAndTransferOnAllocatorToAllocator(targetVector, sourceVector, + startIndex, length, data); } - assertArrayEquals(STR4, targetVector.get(0)); - assertArrayEquals(STR5, targetVector.get(1)); + validateVector.accept(targetVector, data); + } + + final byte [][] binaryData = generateBinaryDataArray(3, 10); + try (final BufferAllocator targetAllocator = allocator.newChildAllocator("target-alloc", 256, maxAllocation); + final ViewVarBinaryVector targetVector = newViewVarBinaryVector("split-target", targetAllocator)) { + try (final BufferAllocator sourceAllocator = allocator.newChildAllocator("source-alloc", 256, maxAllocation); + final ViewVarBinaryVector sourceVector = newViewVarBinaryVector(EMPTY_SCHEMA_PATH, sourceAllocator)) { + testSplitAndTransferOnAllocatorToAllocator(targetVector, sourceVector, + startIndex, length, binaryData); + } + validateVector.accept(targetVector, binaryData); } } @@ -2016,12 +2112,21 @@ public void testSplitAndTransfer7() { * With long strings. */ @Test - public void testSplitAndTransfer8() { + public void testSplitAndTransferWithLongStringsOnAllocatorToAllocator() { final int initialReservation = 1024; // Here we have the target vector being transferred with a long string // hence, the data buffer will be allocated. // The default data buffer allocation takes // BaseVariableWidthViewVector.INITIAL_VIEW_VALUE_ALLOCATION * BaseVariableWidthViewVector.ELEMENT_SIZE + final byte [][] data = new byte[][]{STR1, STR2, STR3}; + final int startIndex = 0; + final int length = 2; + + BiConsumer validateVector = (targetVector, expectedData) -> { + IntStream.range(startIndex, length).forEach(i -> + assertArrayEquals(expectedData[i], targetVector.get(i - startIndex))); + }; + final int maxAllocation = initialReservation + BaseVariableWidthViewVector.INITIAL_VIEW_VALUE_ALLOCATION * BaseVariableWidthViewVector.ELEMENT_SIZE; try (final BufferAllocator targetAllocator = allocator.newChildAllocator("target-alloc", @@ -2030,136 +2135,169 @@ public void testSplitAndTransfer8() { try (final BufferAllocator sourceAllocator = allocator.newChildAllocator("source-alloc", initialReservation, maxAllocation); final ViewVarCharVector sourceVector = newViewVarCharVector(EMPTY_SCHEMA_PATH, sourceAllocator)) { - sourceVector.allocateNew(48, 3); - - sourceVector.set(0, STR1); - sourceVector.set(1, STR2); - sourceVector.set(2, STR3); - sourceVector.setValueCount(3); - - final long allocatedMem = allocator.getAllocatedMemory(); - final int validityRefCnt = sourceVector.getValidityBuffer().refCnt(); - final int dataRefCnt = sourceVector.getDataBuffer().refCnt(); - - sourceVector.splitAndTransferTo(0, 2, targetVector); - // we allocate view and data buffers for the target vector - assertTrue(allocatedMem < allocator.getAllocatedMemory()); - - // the refcnts of each buffer for this test should be the same as what - // the source allocator ended up with. - assertEquals(validityRefCnt, sourceVector.getValidityBuffer().refCnt()); - // since the new view buffer is allocated, the refcnt is the same as the source vector. - assertEquals(dataRefCnt, sourceVector.getDataBuffer().refCnt()); - - assertArrayEquals(STR1, sourceVector.get(0)); - assertArrayEquals(STR2, sourceVector.get(1)); - assertArrayEquals(STR3, sourceVector.get(2)); + testSplitAndTransferOnAllocatorToAllocator(targetVector, sourceVector, + startIndex, length, data); } - assertArrayEquals(STR1, targetVector.get(0)); - assertArrayEquals(STR2, targetVector.get(1)); + validateVector.accept(targetVector, data); } - } - @Test - public void testReallocAfterVectorTransfer1() { - try (final ViewVarCharVector vector = new ViewVarCharVector(EMPTY_SCHEMA_PATH, allocator)) { - /* 4096 values with 16 bytes per record */ - final int bytesPerRecord = 32; - vector.allocateNew(4096 * bytesPerRecord, 4096); - int valueCapacity = vector.getValueCapacity(); - assertTrue(valueCapacity >= 4096); - - /* populate the vector */ - for (int i = 0; i < valueCapacity; i++) { - if ((i & 1) == 1) { - vector.set(i, STR1); - } else { - vector.set(i, STR2); - } + final byte [][] binaryData = generateBinaryDataArray(3, 18); + + try (final BufferAllocator targetAllocator = allocator.newChildAllocator("target-alloc", + initialReservation, maxAllocation); + final ViewVarBinaryVector targetVector = newViewVarBinaryVector("split-target", targetAllocator)) { + try (final BufferAllocator sourceAllocator = allocator.newChildAllocator("source-alloc", + initialReservation, maxAllocation); + final ViewVarBinaryVector sourceVector = newViewVarBinaryVector(EMPTY_SCHEMA_PATH, sourceAllocator)) { + testSplitAndTransferOnAllocatorToAllocator(targetVector, sourceVector, + startIndex, length, binaryData); } + validateVector.accept(targetVector, binaryData); + } + } - /* Check the vector output */ - for (int i = 0; i < valueCapacity; i++) { - if ((i & 1) == 1) { - assertArrayEquals(STR1, vector.get(i)); - } else { - assertArrayEquals(STR2, vector.get(i)); - } + private void testReallocAfterVectorTransferHelper(BaseVariableWidthViewVector vector, + byte[] str1, byte[] str2) { + /* 4096 values with 16 bytes per record */ + final int bytesPerRecord = 32; + vector.allocateNew(4096 * bytesPerRecord, 4096); + int valueCapacity = vector.getValueCapacity(); + assertTrue(valueCapacity >= 4096); + + /* populate the vector */ + for (int i = 0; i < valueCapacity; i++) { + if ((i & 1) == 1) { + vector.set(i, str1); + } else { + vector.set(i, str2); } + } - /* trigger first realloc */ - vector.setSafe(valueCapacity, STR2, 0, STR2.length); - assertTrue(vector.getValueCapacity() >= 2 * valueCapacity); - while (vector.getByteCapacity() < bytesPerRecord * vector.getValueCapacity()) { - vector.reallocViewBuffer(); - vector.reallocViewDataBuffer(); + /* Check the vector output */ + for (int i = 0; i < valueCapacity; i++) { + if ((i & 1) == 1) { + assertArrayEquals(str1, vector.get(i)); + } else { + assertArrayEquals(str2, vector.get(i)); } + } - /* populate the remaining vector */ - for (int i = valueCapacity; i < vector.getValueCapacity(); i++) { - if ((i & 1) == 1) { - vector.set(i, STR1); - } else { - vector.set(i, STR2); - } + /* trigger first realloc */ + vector.setSafe(valueCapacity, str2, 0, str2.length); + assertTrue(vector.getValueCapacity() >= 2 * valueCapacity); + while (vector.getByteCapacity() < bytesPerRecord * vector.getValueCapacity()) { + vector.reallocViewBuffer(); + vector.reallocViewDataBuffer(); + } + + /* populate the remaining vector */ + for (int i = valueCapacity; i < vector.getValueCapacity(); i++) { + if ((i & 1) == 1) { + vector.set(i, str1); + } else { + vector.set(i, str2); } + } - /* Check the vector output */ - valueCapacity = vector.getValueCapacity(); - for (int i = 0; i < valueCapacity; i++) { - if ((i & 1) == 1) { - assertArrayEquals(STR1, vector.get(i)); - } else { - assertArrayEquals(STR2, vector.get(i)); - } + /* Check the vector output */ + valueCapacity = vector.getValueCapacity(); + for (int i = 0; i < valueCapacity; i++) { + if ((i & 1) == 1) { + assertArrayEquals(str1, vector.get(i)); + } else { + assertArrayEquals(str2, vector.get(i)); } + } + + /* trigger second realloc */ + vector.setSafe(valueCapacity + bytesPerRecord, str2, 0, str2.length); + assertTrue(vector.getValueCapacity() >= 2 * valueCapacity); + while (vector.getByteCapacity() < bytesPerRecord * vector.getValueCapacity()) { + vector.reallocViewBuffer(); + vector.reallocViewDataBuffer(); + } - /* trigger second realloc */ - vector.setSafe(valueCapacity + bytesPerRecord, STR2, 0, STR2.length); - assertTrue(vector.getValueCapacity() >= 2 * valueCapacity); - while (vector.getByteCapacity() < bytesPerRecord * vector.getValueCapacity()) { - vector.reallocViewBuffer(); - vector.reallocViewDataBuffer(); + /* populate the remaining vector */ + for (int i = valueCapacity; i < vector.getValueCapacity(); i++) { + if ((i & 1) == 1) { + vector.set(i, str1); + } else { + vector.set(i, str2); } + } - /* populate the remaining vector */ - for (int i = valueCapacity; i < vector.getValueCapacity(); i++) { - if ((i & 1) == 1) { - vector.set(i, STR1); - } else { - vector.set(i, STR2); - } + /* Check the vector output */ + valueCapacity = vector.getValueCapacity(); + for (int i = 0; i < valueCapacity; i++) { + if ((i & 1) == 1) { + assertArrayEquals(str1, vector.get(i)); + } else { + assertArrayEquals(str2, vector.get(i)); } + } - /* Check the vector output */ - valueCapacity = vector.getValueCapacity(); - for (int i = 0; i < valueCapacity; i++) { - if ((i & 1) == 1) { - assertArrayEquals(STR1, vector.get(i)); - } else { - assertArrayEquals(STR2, vector.get(i)); - } + /* We are potentially working with 4x the size of vector buffer + * that we initially started with. + * Now let's transfer the vector. + */ + + TransferPair transferPair = vector.getTransferPair(allocator); + transferPair.transfer(); + BaseVariableWidthViewVector toVector = (BaseVariableWidthViewVector) transferPair.getTo(); + valueCapacity = toVector.getValueCapacity(); + + for (int i = 0; i < valueCapacity; i++) { + if ((i & 1) == 1) { + assertArrayEquals(str1, toVector.get(i)); + } else { + assertArrayEquals(str2, toVector.get(i)); } + } + toVector.close(); + } - /* We are potentially working with 4x the size of vector buffer - * that we initially started with. - * Now let's transfer the vector. - */ + @Test + public void testReallocAfterVectorTransfer() { + try (final ViewVarCharVector vector = new ViewVarCharVector(EMPTY_SCHEMA_PATH, allocator)) { + testReallocAfterVectorTransferHelper(vector, STR1, STR2); + } - TransferPair transferPair = vector.getTransferPair(allocator); - transferPair.transfer(); - ViewVarCharVector toVector = (ViewVarCharVector) transferPair.getTo(); - valueCapacity = toVector.getValueCapacity(); + try (final ViewVarBinaryVector vector = new ViewVarBinaryVector(EMPTY_SCHEMA_PATH, allocator)) { + testReallocAfterVectorTransferHelper(vector, generateRandomBinaryData(12), + generateRandomBinaryData(13)); + } + } - for (int i = 0; i < valueCapacity; i++) { - if ((i & 1) == 1) { - assertArrayEquals(STR1, toVector.get(i)); - } else { - assertArrayEquals(STR2, toVector.get(i)); - } - } + private void testSplitAndTransferWithMultipleDataBuffersHelper(BaseVariableWidthViewVector sourceVector, + BaseVariableWidthViewVector targetVector, int startIndex, int length, byte[][] data) { + sourceVector.allocateNew(48, 4); + + for (int i = 0; i < data.length; i++) { + sourceVector.set(i, data[i]); + } + sourceVector.setValueCount(data.length); + + // we should have multiple data buffers + assertTrue(sourceVector.getDataBuffers().size() > 1); + + final long allocatedMem = allocator.getAllocatedMemory(); + final int validityRefCnt = sourceVector.getValidityBuffer().refCnt(); + final int dataRefCnt = sourceVector.getDataBuffer().refCnt(); - toVector.close(); + // split and transfer with slice starting at the beginning: + // this should not allocate anything new + sourceVector.splitAndTransferTo(startIndex, length, targetVector); + // we allocate view and data buffers for the target vector + assertTrue(allocatedMem < allocator.getAllocatedMemory()); + + // the refcnts of each buffer for this test should be the same as what + // the source allocator ended up with. + assertEquals(validityRefCnt, sourceVector.getValidityBuffer().refCnt()); + // since the new view buffer is allocated, the refcnt is the same as the source vector. + assertEquals(dataRefCnt, sourceVector.getDataBuffer().refCnt()); + + for (int i = 0; i < data.length; i++) { + assertArrayEquals(data[i], sourceVector.get(i)); } } @@ -2171,45 +2309,31 @@ public void testReallocAfterVectorTransfer1() { * Check multi-data buffer source copying */ @Test - public void testSplitAndTransfer9() { + public void testSplitAndTransferWithMultipleDataBuffers() { + final String str4 = generateRandomString(35); + final byte[][] data = new byte[][]{STR1, STR2, STR3, str4.getBytes(StandardCharsets.UTF_8)}; + final int startIndex = 1; + final int length = 3; + + BiConsumer validateVector = (targetVector, expectedData) -> { + IntStream.range(startIndex, length).forEach(i -> + assertArrayEquals(expectedData[i], targetVector.get(i - startIndex))); + }; + try (final ViewVarCharVector targetVector = new ViewVarCharVector("target", allocator)) { - String str4 = generateRandomString(35); try (final ViewVarCharVector sourceVector = new ViewVarCharVector("source", allocator)) { - sourceVector.allocateNew(48, 4); - - sourceVector.set(0, STR1); - sourceVector.set(1, STR2); - sourceVector.set(2, STR3); - sourceVector.set(3, str4.getBytes(StandardCharsets.UTF_8)); - sourceVector.setValueCount(4); - - // we should have multiple data buffers - assertTrue(sourceVector.getDataBuffers().size() > 1); - - final long allocatedMem = allocator.getAllocatedMemory(); - final int validityRefCnt = sourceVector.getValidityBuffer().refCnt(); - final int dataRefCnt = sourceVector.getDataBuffer().refCnt(); - - // split and transfer with slice starting at the beginning: - // this should not allocate anything new - sourceVector.splitAndTransferTo(1, 3, targetVector); - // we allocate view and data buffers for the target vector - assertTrue(allocatedMem < allocator.getAllocatedMemory()); - - // the refcnts of each buffer for this test should be the same as what - // the source allocator ended up with. - assertEquals(validityRefCnt, sourceVector.getValidityBuffer().refCnt()); - // since the new view buffer is allocated, the refcnt is the same as the source vector. - assertEquals(dataRefCnt, sourceVector.getDataBuffer().refCnt()); - - assertArrayEquals(STR1, sourceVector.get(0)); - assertArrayEquals(STR2, sourceVector.get(1)); - assertArrayEquals(STR3, sourceVector.get(2)); - assertArrayEquals(str4.getBytes(StandardCharsets.UTF_8), sourceVector.get(3)); + testSplitAndTransferWithMultipleDataBuffersHelper(sourceVector, targetVector, + startIndex, length, data); + } + validateVector.accept(targetVector, data); + } + + try (final ViewVarBinaryVector targetVector = new ViewVarBinaryVector("target", allocator)) { + try (final ViewVarBinaryVector sourceVector = new ViewVarBinaryVector("source", allocator)) { + testSplitAndTransferWithMultipleDataBuffersHelper(sourceVector, targetVector, + startIndex, length, data); } - assertArrayEquals(STR2, targetVector.get(0)); - assertArrayEquals(STR3, targetVector.get(1)); - assertArrayEquals(str4.getBytes(StandardCharsets.UTF_8), targetVector.get(2)); + validateVector.accept(targetVector, data); } } From 67c6df1f2addd39b77e0e66e999164acff3d2ae3 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 6 Jun 2024 17:40:49 +0900 Subject: [PATCH 72/93] GH-41964: [CI][C++] Clear cache for mamba on AppVeyor (#41977) ### Rationale for this change It seems that mamba may use invalid download URL when there are invalid caches. ### What changes are included in this PR? Clear caches explicitly. ### Are these changes tested? Yes. ### Are there any user-facing changes? No. * GitHub Issue: #41964 Authored-by: Sutou Kouhei Signed-off-by: Antoine Pitrou --- ci/appveyor-cpp-setup.bat | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/appveyor-cpp-setup.bat b/ci/appveyor-cpp-setup.bat index 5c4a11832d5ee..5a9dffa166fb7 100644 --- a/ci/appveyor-cpp-setup.bat +++ b/ci/appveyor-cpp-setup.bat @@ -66,6 +66,9 @@ set CONDA_PACKAGES=%CONDA_PACKAGES% --file=ci\conda_env_cpp.txt @rem Force conda to use conda-forge conda config --add channels conda-forge conda config --remove channels defaults +@rem Ensure using the latest information. If there are invalid caches, +@rem mamba may use invalid download URL. +mamba clean --all -y @rem Arrow conda environment mamba create -n arrow -y -c conda-forge ^ --file=ci\conda_env_python.txt ^ From 374b8f6ddec3b7614408ea874ffb29981c2a295d Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 6 Jun 2024 18:40:30 +0900 Subject: [PATCH 73/93] GH-41903: [CI][GLib] Use the latest Ruby to use OpenSSL 3 (#42001) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change Old Ruby ships OpenSSL 1 but googld-cloud-cpp requires OpenSSL 3. We need to use Ruby that ships OpenSSL 3. ### What changes are included in this PR? Use the latest Ruby. ### Are these changes tested? Yes. ### Are there any user-facing changes? No. * GitHub Issue: #41903 Authored-by: Sutou Kouhei Signed-off-by: Raúl Cumplido --- .github/workflows/ruby.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 35c4460d47bc6..eb00bc5f92a8d 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -197,9 +197,7 @@ jobs: mingw-n-bits: - 64 ruby-version: - # TODO: Use the latest Ruby again when we fix GH-39130. - # - ruby - - "3.1" + - ruby env: ARROW_BUILD_STATIC: OFF ARROW_BUILD_TESTS: OFF From cbb0e4cb06fe2cc93295a6b6a9d81dad047feb61 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Jun 2024 11:18:46 -0400 Subject: [PATCH 74/93] MINOR: [JS] Bump @typescript-eslint/eslint-plugin from 7.11.0 to 7.12.0 in /js (#41950) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [@typescript-eslint/eslint-plugin](https://github.com/typescript-eslint/typescript-eslint/tree/HEAD/packages/eslint-plugin) from 7.11.0 to 7.12.0.
Release notes

Sourced from @​typescript-eslint/eslint-plugin's releases.

v7.12.0

7.12.0 (2024-06-03)

🚀 Features

  • eslint-plugin: [no-useless-template-literals] rename to no-useless-template-expression (deprecate no-useless-template-literals) (#8821)
  • eslint-plugin: [no-floating-promises] add option 'allowForKnownSafePromises' (#9186)
  • rule-tester: check for parsing errors in suggestion fixes (#9052)
  • rule-tester: port checkDuplicateTestCases from ESLint (#9026)

🩹 Fixes

  • no-useless-template-expression -> no-unnecessary-template-expression (#9174)
  • eslint-plugin: [no-unnecessary-type-assertion] combine template literal check with const variable check (#8820)
  • eslint-plugin: [dot-notation] fix false positive when accessing private/protected property with optional chaining (#8851)
  • eslint-plugin: [explicit-member-accessibility] refine report locations (#8869)
  • eslint-plugin: [no-unnecessary-type-assertion] declares are always defined, so always check declares (#8901)
  • eslint-plugin: [prefer-literal-enum-member] allow using member it self on allowBitwiseExpressions (#9114)
  • eslint-plugin: [return-await] clean up in-try-catch detection and make autofixes safe (#9031)
  • eslint-plugin: [member-ordering] also TSMethodSignature can be get/set (#9193)
  • types: correct typing ParserOptions (#9202)

❤️ Thank You

You can read about our versioning strategy and releases on our website.

Changelog

Sourced from @​typescript-eslint/eslint-plugin's changelog.

7.12.0 (2024-06-03)

🚀 Features

  • eslint-plugin: [no-useless-template-literals] rename to no-useless-template-expression (deprecate no-useless-template-literals)

  • rule-tester: check for parsing errors in suggestion fixes

  • rule-tester: port checkDuplicateTestCases from ESLint

  • eslint-plugin: [no-floating-promises] add option 'allowForKnownSafePromises'

🩹 Fixes

  • no-useless-template-expression -> no-unnecessary-template-expression

  • eslint-plugin: [no-unnecessary-type-assertion] combine template literal check with const variable check

  • eslint-plugin: [dot-notation] fix false positive when accessing private/protected property with optional chaining

  • eslint-plugin: [explicit-member-accessibility] refine report locations

  • eslint-plugin: [no-unnecessary-type-assertion] declares are always defined, so always check declares

  • eslint-plugin: [prefer-literal-enum-member] allow using member it self on allowBitwiseExpressions

  • eslint-plugin: [return-await] clean up in-try-catch detection and make autofixes safe

  • eslint-plugin: [member-ordering] also TSMethodSignature can be get/set

❤️ Thank You

  • Abraham Guo
  • Han Yeong-woo
  • Joshua Chen
  • Kim Sang Du
  • Kirk Waiblinger
  • YeonJuan

You can read about our versioning strategy and releases on our website.

Commits
  • 7e93b28 chore(release): publish 7.12.0
  • d0adcf1 docs: clarify what require-await does (#9200)
  • 04990d5 feat(eslint-plugin): [no-floating-promises] add option 'allowForKnownSafeProm...
  • ad85249 docs: mention related ESLint rules in no-unused-vars page (#9198)
  • e80a8d6 docs: improve description for no-dynamic-delete (#9195)
  • 9f92b30 docs: explicitly mention unbound-method limitation with thisArg (#9197)
  • 08a9448 docs: add example with PascalCase function components (#9196)
  • 5ca7f6e feat(rule-tester): port checkDuplicateTestCases from ESLint (#9026)
  • a9dd526 fix(eslint-plugin): [member-ordering] also TSMethodSignature can be get/set (...
  • 2619c3b fix(eslint-plugin): [return-await] clean up in-try-catch detection and make a...
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=@typescript-eslint/eslint-plugin&package-manager=npm_and_yarn&previous-version=7.11.0&new-version=7.12.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- js/package.json | 2 +- js/yarn.lock | 77 ++++++++++++++----------------------------------- 2 files changed, 22 insertions(+), 57 deletions(-) diff --git a/js/package.json b/js/package.json index e7be3c80d82c9..ecb6d3a366f7e 100644 --- a/js/package.json +++ b/js/package.json @@ -72,7 +72,7 @@ "@types/glob": "8.1.0", "@types/jest": "29.5.12", "@types/multistream": "4.1.3", - "@typescript-eslint/eslint-plugin": "7.11.0", + "@typescript-eslint/eslint-plugin": "7.12.0", "@typescript-eslint/parser": "7.12.0", "async-done": "2.0.0", "benny": "3.7.1", diff --git a/js/yarn.lock b/js/yarn.lock index 3cf3284a9f306..5ab52beaf8f15 100644 --- a/js/yarn.lock +++ b/js/yarn.lock @@ -1434,16 +1434,16 @@ dependencies: "@types/yargs-parser" "*" -"@typescript-eslint/eslint-plugin@7.11.0": - version "7.11.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.11.0.tgz#f90f0914657ead08e1c75f66939c926edeab42dd" - integrity sha512-P+qEahbgeHW4JQ/87FuItjBj8O3MYv5gELDzr8QaQ7fsll1gSMTYb6j87MYyxwf3DtD7uGFB9ShwgmCJB5KmaQ== +"@typescript-eslint/eslint-plugin@7.12.0": + version "7.12.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.12.0.tgz#f87a32e8972b8a60024f2f8f12205e7c8108bc41" + integrity sha512-7F91fcbuDf/d3S8o21+r3ZncGIke/+eWk0EpO21LXhDfLahriZF9CGj4fbAetEjlaBdjdSm9a6VeXbpbT6Z40Q== dependencies: "@eslint-community/regexpp" "^4.10.0" - "@typescript-eslint/scope-manager" "7.11.0" - "@typescript-eslint/type-utils" "7.11.0" - "@typescript-eslint/utils" "7.11.0" - "@typescript-eslint/visitor-keys" "7.11.0" + "@typescript-eslint/scope-manager" "7.12.0" + "@typescript-eslint/type-utils" "7.12.0" + "@typescript-eslint/utils" "7.12.0" + "@typescript-eslint/visitor-keys" "7.12.0" graphemer "^1.4.0" ignore "^5.3.1" natural-compare "^1.4.0" @@ -1460,14 +1460,6 @@ "@typescript-eslint/visitor-keys" "7.12.0" debug "^4.3.4" -"@typescript-eslint/scope-manager@7.11.0": - version "7.11.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-7.11.0.tgz#cf5619b01de62a226a59add15a02bde457335d1d" - integrity sha512-27tGdVEiutD4POirLZX4YzT180vevUURJl4wJGmm6TrQoiYwuxTIY98PBp6L2oN+JQxzE0URvYlzJaBHIekXAw== - dependencies: - "@typescript-eslint/types" "7.11.0" - "@typescript-eslint/visitor-keys" "7.11.0" - "@typescript-eslint/scope-manager@7.12.0": version "7.12.0" resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-7.12.0.tgz#259c014362de72dd34f995efe6bd8dda486adf58" @@ -1476,40 +1468,21 @@ "@typescript-eslint/types" "7.12.0" "@typescript-eslint/visitor-keys" "7.12.0" -"@typescript-eslint/type-utils@7.11.0": - version "7.11.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-7.11.0.tgz#ac216697d649084fedf4a910347b9642bd0ff099" - integrity sha512-WmppUEgYy+y1NTseNMJ6mCFxt03/7jTOy08bcg7bxJJdsM4nuhnchyBbE8vryveaJUf62noH7LodPSo5Z0WUCg== +"@typescript-eslint/type-utils@7.12.0": + version "7.12.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-7.12.0.tgz#9dfaaa1972952f395ec5be4f5bbfc4d3cdc63908" + integrity sha512-lib96tyRtMhLxwauDWUp/uW3FMhLA6D0rJ8T7HmH7x23Gk1Gwwu8UZ94NMXBvOELn6flSPiBrCKlehkiXyaqwA== dependencies: - "@typescript-eslint/typescript-estree" "7.11.0" - "@typescript-eslint/utils" "7.11.0" + "@typescript-eslint/typescript-estree" "7.12.0" + "@typescript-eslint/utils" "7.12.0" debug "^4.3.4" ts-api-utils "^1.3.0" -"@typescript-eslint/types@7.11.0": - version "7.11.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-7.11.0.tgz#5e9702a5e8b424b7fc690e338d359939257d6722" - integrity sha512-MPEsDRZTyCiXkD4vd3zywDCifi7tatc4K37KqTprCvaXptP7Xlpdw0NR2hRJTetG5TxbWDB79Ys4kLmHliEo/w== - "@typescript-eslint/types@7.12.0": version "7.12.0" resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-7.12.0.tgz#bf208f971a8da1e7524a5d9ae2b5f15192a37981" integrity sha512-o+0Te6eWp2ppKY3mLCU+YA9pVJxhUJE15FV7kxuD9jgwIAa+w/ycGJBMrYDTpVGUM/tgpa9SeMOugSabWFq7bg== -"@typescript-eslint/typescript-estree@7.11.0": - version "7.11.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-7.11.0.tgz#7cbc569bc7336c3a494ceaf8204fdee5d5dbb7fa" - integrity sha512-cxkhZ2C/iyi3/6U9EPc5y+a6csqHItndvN/CzbNXTNrsC3/ASoYQZEt9uMaEp+xFNjasqQyszp5TumAVKKvJeQ== - dependencies: - "@typescript-eslint/types" "7.11.0" - "@typescript-eslint/visitor-keys" "7.11.0" - debug "^4.3.4" - globby "^11.1.0" - is-glob "^4.0.3" - minimatch "^9.0.4" - semver "^7.6.0" - ts-api-utils "^1.3.0" - "@typescript-eslint/typescript-estree@7.12.0": version "7.12.0" resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-7.12.0.tgz#e6c1074f248b3db6573ab6a7c47a39c4cd498ff9" @@ -1524,23 +1497,15 @@ semver "^7.6.0" ts-api-utils "^1.3.0" -"@typescript-eslint/utils@7.11.0", "@typescript-eslint/utils@^6.0.0 || ^7.0.0": - version "7.11.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-7.11.0.tgz#524f047f2209959424c3ef689b0d83b3bc09919c" - integrity sha512-xlAWwPleNRHwF37AhrZurOxA1wyXowW4PqVXZVUNCLjB48CqdPJoJWkrpH2nij9Q3Lb7rtWindtoXwxjxlKKCA== +"@typescript-eslint/utils@7.12.0", "@typescript-eslint/utils@^6.0.0 || ^7.0.0": + version "7.12.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-7.12.0.tgz#c6e58fd7f724cdccc848f71e388ad80cbdb95dd0" + integrity sha512-Y6hhwxwDx41HNpjuYswYp6gDbkiZ8Hin9Bf5aJQn1bpTs3afYY4GX+MPYxma8jtoIV2GRwTM/UJm/2uGCVv+DQ== dependencies: "@eslint-community/eslint-utils" "^4.4.0" - "@typescript-eslint/scope-manager" "7.11.0" - "@typescript-eslint/types" "7.11.0" - "@typescript-eslint/typescript-estree" "7.11.0" - -"@typescript-eslint/visitor-keys@7.11.0": - version "7.11.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-7.11.0.tgz#2c50cd292e67645eec05ac0830757071b4a4d597" - integrity sha512-7syYk4MzjxTEk0g/w3iqtgxnFQspDJfn6QKD36xMuuhTzjcxY7F8EmBLnALjVyaOF1/bVocu3bS/2/F7rXrveQ== - dependencies: - "@typescript-eslint/types" "7.11.0" - eslint-visitor-keys "^3.4.3" + "@typescript-eslint/scope-manager" "7.12.0" + "@typescript-eslint/types" "7.12.0" + "@typescript-eslint/typescript-estree" "7.12.0" "@typescript-eslint/visitor-keys@7.12.0": version "7.12.0" From 93712bfc71a5013231b950b2b655d77b14f83fa7 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Thu, 6 Jun 2024 17:38:48 +0200 Subject: [PATCH 75/93] GH-41691: [Doc] Remove notion of "logical type" (#41958) In several places in the Arrow specification and documentation we use the term "logical types", but we don't use it consistently and we don't actually have physical types (only physical layouts) to contrast it with. This creates confusion for readers as it is not immediately clear whether all data types are "logical" and if there is a meaningful distinction behind our usage of this term. Also address GH-14752 by adding a table of data types with their respective parameters and the corresponding layouts. * GitHub Issue: #41691 Authored-by: Antoine Pitrou Signed-off-by: Antoine Pitrou --- docs/source/format/Columnar.rst | 160 ++++++++++++++++++++----- docs/source/format/Versioning.rst | 2 +- docs/source/python/data.rst | 14 +-- docs/source/python/extending_types.rst | 2 +- 4 files changed, 138 insertions(+), 40 deletions(-) diff --git a/docs/source/format/Columnar.rst b/docs/source/format/Columnar.rst index 7c853de7829be..c4dc772808af4 100644 --- a/docs/source/format/Columnar.rst +++ b/docs/source/format/Columnar.rst @@ -70,21 +70,131 @@ concepts, here is a small glossary to help disambiguate. without taking into account any value semantics. For example, a 32-bit signed integer array and 32-bit floating point array have the same layout. -* **Parent** and **child arrays**: names to express relationships - between physical value arrays in a nested type structure. For - example, a ``List``-type parent array has a T-type array as its - child (see more on lists below). +* **Data type**: An application-facing semantic value type that is + implemented using some physical layout. For example, Decimal128 + values are stored as 16 bytes in a fixed-size binary + layout. A timestamp may be stored as 64-bit fixed-size layout. * **Primitive type**: a data type having no child types. This includes such types as fixed bit-width, variable-size binary, and null types. * **Nested type**: a data type whose full structure depends on one or more other child types. Two fully-specified nested types are equal if and only if their child types are equal. For example, ``List`` is distinct from ``List`` iff U and V are different types. -* **Logical type**: An application-facing semantic value type that is - implemented using some physical layout. For example, Decimal - values are stored as 16 bytes in a fixed-size binary - layout. Similarly, strings can be stored as ``List<1-byte>``. A - timestamp may be stored as 64-bit fixed-size layout. +* **Parent** and **child arrays**: names to express relationships + between physical value arrays in a nested type structure. For + example, a ``List``-type parent array has a T-type array as its + child (see more on lists below). +* **Parametric type**: a type which requires additional parameters + for full determination of its semantics. For example, all nested types + are parametric by construction. A timestamp is also parametric as it needs + a unit (such as microseconds) and a timezone. + +Data Types +========== + +The file `Schema.fbs`_ defines built-in data types supported by the +Arrow columnar format. Each data type uses a well-defined physical layout. + +`Schema.fbs`_ is the authoritative source for the description of the +standard Arrow data types. However, we also provide the below table for +convenience: + ++--------------------+------------------------------+------------------------------------------------------------+ +| Type | Type Parameters *(1)* | Physical Memory Layout | ++====================+==============================+============================================================+ +| Null | | Null | ++--------------------+------------------------------+------------------------------------------------------------+ +| Boolean | | Fixed-size Primitive | ++--------------------+------------------------------+------------------------------------------------------------+ +| Int | * bit width | *" (same as above)* | +| | * signedness | | ++--------------------+------------------------------+------------------------------------------------------------+ +| Floating Point | * precision | *"* | ++--------------------+------------------------------+------------------------------------------------------------+ +| Decimal | * bit width | *"* | +| | * scale | | +| | * precision | | ++--------------------+------------------------------+------------------------------------------------------------+ +| Date | * unit | *"* | ++--------------------+------------------------------+------------------------------------------------------------+ +| Time | * bit width *(2)* | *"* | +| | * unit | | ++--------------------+------------------------------+------------------------------------------------------------+ +| Timestamp | * unit | *"* | +| | * timezone | | ++--------------------+------------------------------+------------------------------------------------------------+ +| Interval | * unit | *"* | ++--------------------+------------------------------+------------------------------------------------------------+ +| Duration | * unit | *"* | ++--------------------+------------------------------+------------------------------------------------------------+ +| Fixed-Size Binary | * byte width | Fixed-size Binary | ++--------------------+------------------------------+------------------------------------------------------------+ +| Binary | | Variable-size Binary with 32-bit offsets | ++--------------------+------------------------------+------------------------------------------------------------+ +| Utf8 | | *"* | ++--------------------+------------------------------+------------------------------------------------------------+ +| Large Binary | | Variable-size Binary with 64-bit offsets | ++--------------------+------------------------------+------------------------------------------------------------+ +| Large Utf8 | | *"* | ++--------------------+------------------------------+------------------------------------------------------------+ +| Binary View | | Variable-size Binary View | ++--------------------+------------------------------+------------------------------------------------------------+ +| Utf8 View | | *"* | ++--------------------+------------------------------+------------------------------------------------------------+ +| Fixed-Size List | * *value type* | Fixed-size List | +| | * list size | | ++--------------------+------------------------------+------------------------------------------------------------+ +| List | * *value type* | Variable-size List with 32-bit offsets | ++--------------------+------------------------------+------------------------------------------------------------+ +| Large List | * *value type* | Variable-size List with 64-bit offsets | ++--------------------+------------------------------+------------------------------------------------------------+ +| List View | * *value type* | Variable-size List View with 32-bit offsets and sizes | ++--------------------+------------------------------+------------------------------------------------------------+ +| Large List View | * *value type* | Variable-size List View with 64-bit offsets and sizes | ++--------------------+------------------------------+------------------------------------------------------------+ +| Struct | * *children* | Struct | ++--------------------+------------------------------+------------------------------------------------------------+ +| Map | * *children* | Variable-size List of Structs | +| | * keys sortedness | | ++--------------------+------------------------------+------------------------------------------------------------+ +| Union | * *children* | Dense or Sparse Union *(3)* | +| | * mode | | +| | * type ids | | ++--------------------+------------------------------+------------------------------------------------------------+ +| Dictionary | * *index type* *(4)* | Dictionary Encoded | +| | * *value type* | | +| | * orderedness | | ++--------------------+------------------------------+------------------------------------------------------------+ +| Run-End Encoded | * *run end type* *(5)* | Run-End Encoded | +| | * *value type* | | ++--------------------+------------------------------+------------------------------------------------------------+ + +* \(1) Type parameters listed in *italics* denote a data type's child types. + +* \(2) The *bit width* parameter of a Time type is technically redundant as + each *unit* mandates a single bit width. + +* \(3) Whether a Union type uses the Sparse or Dense layout is denoted by its + *mode* parameter. + +* \(4) The *index type* of a Dictionary type can only be an integer type, + preferably signed, with width 8 to 64 bits. + +* \(5) The *run end type* of a Run-End Encoded type can only be a signed integer type + with width 16 to 64 bits. + +.. note:: + Sometimes the term "logical type" is used to denote the Arrow data types + and distinguish them from their respective physical layouts. However, + unlike other type systems such as `Apache Parquet `__'s, + the Arrow type system doesn't have separate notions of physical types and + logical types. + + The Arrow type system separately provides + :ref:`extension types `, which allow + annotating standard Arrow data types with richer application-facing semantics + (for example defining a "JSON" type laid upon the standard String data type). + .. _format_layout: @@ -93,7 +203,7 @@ Physical Memory Layout Arrays are defined by a few pieces of metadata and data: -* A logical data type. +* A data type. * A sequence of buffers. * A length as a 64-bit signed integer. Implementations are permitted to be limited to 32-bit lengths, see more on this below. @@ -103,8 +213,8 @@ Arrays are defined by a few pieces of metadata and data: Nested arrays additionally have a sequence of one or more sets of these items, called the **child arrays**. -Each logical data type has a well-defined physical layout. Here are -the different physical layouts defined by Arrow: +Each data type has a well-defined physical layout. Here are the different +physical layouts defined by Arrow: * **Primitive (fixed-size)**: a sequence of values each having the same byte or bit width @@ -138,7 +248,7 @@ the different physical layouts defined by Arrow: * **Run-End Encoded (REE)**: a nested layout consisting of two child arrays, one representing values, and one representing the logical index where the run of a corresponding value ends. -* **Null**: a sequence of all null values, having null logical type +* **Null**: a sequence of all null values. The Arrow columnar memory layout only applies to *data* and not *metadata*. Implementations are free to represent metadata in-memory @@ -313,7 +423,7 @@ arrays have a single values buffer, variable-size binary have an **offsets** buffer and **data** buffer. The offsets buffer contains ``length + 1`` signed integers (either -32-bit or 64-bit, depending on the logical type), which encode the +32-bit or 64-bit, depending on the data type), which encode the start position of each slot in the data buffer. The length of the value in each slot is computed using the difference between the offset at that slot's index and the subsequent offset. For example, the @@ -1070,17 +1180,6 @@ of memory buffers for each layout. "Dictionary-encoded",validity,data (indices),, "Run-end encoded",,,, -Logical Types -============= - -The `Schema.fbs`_ defines built-in logical types supported by the -Arrow columnar format. Each logical type uses one of the above -physical layouts. Nested logical types may have different physical -layouts depending on the particular realization of the type. - -We do not go into detail about the logical types definitions in this -document as we consider `Schema.fbs`_ to be authoritative. - .. _format-ipc: Serialization and Interprocess Communication (IPC) @@ -1160,17 +1259,16 @@ Schema message -------------- The Flatbuffers files `Schema.fbs`_ contains the definitions for all -built-in logical data types and the ``Schema`` metadata type which -represents the schema of a given record batch. A schema consists of -an ordered sequence of fields, each having a name and type. A -serialized ``Schema`` does not contain any data buffers, only type -metadata. +built-in data types and the ``Schema`` metadata type which represents +the schema of a given record batch. A schema consists of an ordered +sequence of fields, each having a name and type. A serialized ``Schema`` +does not contain any data buffers, only type metadata. The ``Field`` Flatbuffers type contains the metadata for a single array. This includes: * The field's name -* The field's logical type +* The field's data type * Whether the field is semantically nullable. While this has no bearing on the array's physical layout, many systems distinguish nullable and non-nullable fields and we want to allow them to diff --git a/docs/source/format/Versioning.rst b/docs/source/format/Versioning.rst index 7ba01107074d0..8fcf11b21f0cc 100644 --- a/docs/source/format/Versioning.rst +++ b/docs/source/format/Versioning.rst @@ -51,7 +51,7 @@ data. An increase in the **minor** version of the format version, such as 1.0.0 to 1.1.0, indicates that 1.1.0 contains new features not available in 1.0.0. So long as these features are not used (such as a -new logical data type), forward compatibility is preserved. +new data type), forward compatibility is preserved. Long-Term Stability =================== diff --git a/docs/source/python/data.rst b/docs/source/python/data.rst index 598c8c125fb83..4a0f2af6d4868 100644 --- a/docs/source/python/data.rst +++ b/docs/source/python/data.rst @@ -26,8 +26,8 @@ with memory buffers, like the ones explained in the documentation on :ref:`Memory and IO `. These data structures are exposed in Python through a series of interrelated classes: -* **Type Metadata**: Instances of ``pyarrow.DataType``, which describe a logical - array type +* **Type Metadata**: Instances of ``pyarrow.DataType``, which describe the + type of an array and govern how its values are interpreted * **Schemas**: Instances of ``pyarrow.Schema``, which describe a named collection of types. These can be thought of as the column types in a table-like object. @@ -55,8 +55,8 @@ array data. These include: * **Nested types**: list, map, struct, and union * **Dictionary type**: An encoded categorical type (more on this later) -Each logical data type in Arrow has a corresponding factory function for -creating an instance of that type object in Python: +Each data type in Arrow has a corresponding factory function for creating +an instance of that type object in Python: .. ipython:: python @@ -72,9 +72,9 @@ creating an instance of that type object in Python: print(t4) print(t5) -We use the name **logical type** because the **physical** storage may be the -same for one or more types. For example, ``int64``, ``float64``, and -``timestamp[ms]`` all occupy 64 bits per value. +.. note:: + Different data types might use a given physical storage. For example, + ``int64``, ``float64``, and ``timestamp[ms]`` all occupy 64 bits per value. These objects are ``metadata``; they are used for describing the data in arrays, schemas, and record batches. In Python, they can be used in functions where the diff --git a/docs/source/python/extending_types.rst b/docs/source/python/extending_types.rst index 83fce84f47c08..d746505348157 100644 --- a/docs/source/python/extending_types.rst +++ b/docs/source/python/extending_types.rst @@ -118,7 +118,7 @@ Defining extension types ("user-defined types") Arrow has the notion of extension types in the metadata specification as a possibility to extend the built-in types. This is done by annotating any of the -built-in Arrow logical types (the "storage type") with a custom type name and +built-in Arrow data types (the "storage type") with a custom type name and optional serialized representation ("ARROW:extension:name" and "ARROW:extension:metadata" keys in the Field’s custom_metadata of an IPC message). From 164be4882176c5f84eb7cde52b98d69a72fe7ea8 Mon Sep 17 00:00:00 2001 From: Felipe Oliveira Carvalho Date: Thu, 6 Jun 2024 14:24:47 -0300 Subject: [PATCH 76/93] GH-41994 [C++]: kernel.cc: Remove defaults on switch so that compiler can check full enum coverage for us (#41995) ### Rationale for this change To let the compiler warn us about missing cases and make the non-handled cases more obvious. ### What changes are included in this PR? Removal of `default` in the switches and improving some dchecks with a message. ### Are these changes tested? By existing tests. * GitHub Issue: #41994 Authored-by: Felipe Oliveira Carvalho Signed-off-by: Felipe Oliveira Carvalho --- cpp/src/arrow/compute/kernel.cc | 42 ++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/cpp/src/arrow/compute/kernel.cc b/cpp/src/arrow/compute/kernel.cc index 9cc5cc10917ee..5c87ef2cd0561 100644 --- a/cpp/src/arrow/compute/kernel.cc +++ b/cpp/src/arrow/compute/kernel.cc @@ -361,7 +361,8 @@ size_t InputType::Hash() const { case InputType::EXACT_TYPE: hash_combine(result, type_->Hash()); break; - default: + case InputType::ANY_TYPE: + case InputType::USE_TYPE_MATCHER: break; } return result; @@ -378,10 +379,8 @@ std::string InputType::ToString() const { break; case InputType::USE_TYPE_MATCHER: { ss << type_matcher_->ToString(); - } break; - default: - DCHECK(false); break; + } } return ss.str(); } @@ -400,9 +399,8 @@ bool InputType::Equals(const InputType& other) const { return type_->Equals(*other.type_); case InputType::USE_TYPE_MATCHER: return type_matcher_->Equals(*other.type_matcher_); - default: - return false; } + return false; } bool InputType::Matches(const DataType& type) const { @@ -411,21 +409,23 @@ bool InputType::Matches(const DataType& type) const { return type_->Equals(type); case InputType::USE_TYPE_MATCHER: return type_matcher_->Matches(type); - default: - // ANY_TYPE + case InputType::ANY_TYPE: return true; } + return false; } bool InputType::Matches(const Datum& value) const { switch (value.kind()) { + case Datum::NONE: + case Datum::RECORD_BATCH: + case Datum::TABLE: + DCHECK(false) << "Matches expects ARRAY, CHUNKED_ARRAY or SCALAR"; + return false; case Datum::ARRAY: case Datum::CHUNKED_ARRAY: case Datum::SCALAR: break; - default: - DCHECK(false); - return false; } return Matches(*value.type()); } @@ -445,11 +445,13 @@ const TypeMatcher& InputType::type_matcher() const { Result OutputType::Resolve(KernelContext* ctx, const std::vector& types) const { - if (kind_ == OutputType::FIXED) { - return type_.get(); - } else { - return resolver_(ctx, types); + switch (kind_) { + case OutputType::FIXED: + return type_; + case OutputType::COMPUTED: + break; } + return resolver_(ctx, types); } const std::shared_ptr& OutputType::type() const { @@ -463,11 +465,13 @@ const OutputType::Resolver& OutputType::resolver() const { } std::string OutputType::ToString() const { - if (kind_ == OutputType::FIXED) { - return type_->ToString(); - } else { - return "computed"; + switch (kind_) { + case OutputType::FIXED: + return type_->ToString(); + case OutputType::COMPUTED: + break; } + return "computed"; } // ---------------------------------------------------------------------- From 41ae29ebd98e66d1502d6a830f88d6da056c670e Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Fri, 7 Jun 2024 10:15:50 +0900 Subject: [PATCH 77/93] GH-42005: [Java][Integration][CI] Fix ARROW_BUILD_ROOT Path to find pom.xml (#42008) ### Rationale for this change This PR aims to fix the issue where the integration tests are failing due to the missing `/java/pom.xml` file. It appears that the current code incorrectly determines the path to `ARROW_BUILD_ROOT`, leading to the failure in locating the `pom.xml` file. ### What changes are included in this PR? - Updating the `ARROW_BUILD_ROOT` path determination logic in `tester_java.py` to correctly reference the project root. ### Are these changes tested? Maybe, Yes. ### Are there any user-facing changes? No. * GitHub Issue: #42005 Authored-by: Hyunseok Seo Signed-off-by: Sutou Kouhei --- dev/archery/archery/integration/tester_java.py | 2 +- dev/archery/archery/integration/tester_js.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/archery/archery/integration/tester_java.py b/dev/archery/archery/integration/tester_java.py index ccc807410a848..9b14c6939cde8 100644 --- a/dev/archery/archery/integration/tester_java.py +++ b/dev/archery/archery/integration/tester_java.py @@ -28,7 +28,7 @@ ARROW_BUILD_ROOT = os.environ.get( 'ARROW_BUILD_ROOT', - Path(__file__).resolve().parents[5] + Path(__file__).resolve().parents[4] ) diff --git a/dev/archery/archery/integration/tester_js.py b/dev/archery/archery/integration/tester_js.py index 3d1a229931cde..dcf56f9a5ab6b 100644 --- a/dev/archery/archery/integration/tester_js.py +++ b/dev/archery/archery/integration/tester_js.py @@ -24,7 +24,7 @@ ARROW_BUILD_ROOT = os.environ.get( 'ARROW_BUILD_ROOT', - Path(__file__).resolve().parents[5] + Path(__file__).resolve().parents[4] ) ARROW_JS_ROOT = os.path.join(ARROW_BUILD_ROOT, 'js') _EXE_PATH = os.path.join(ARROW_JS_ROOT, 'bin') From a708fabfe6f90a890978d8f026c70cdf18caf251 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 6 Jun 2024 19:25:36 -0700 Subject: [PATCH 78/93] MINOR: [C++] Include `` before using `std::string` (#42004) ### Rationale for this change I work on MSVC's STL, and we regularly build popular open-source projects, including yours, with development builds of the MSVC toolset. This allows us to find and fix toolset regressions before they affect users, and also allows us to provide advance notice of breaking changes, which is the case here. We recently merged https://github.com/microsoft/STL/pull/4633 which will ship in VS 2022 17.11 Preview 3. This improved build throughput by refactoring `` so that it no longer drags in `std::string`. It's also a source-breaking change for code that wasn't properly including ``. Your `cpp/src/arrow/json/object_writer.h` declares `std::string Serialize();` without including ``. When built with our updated STL, this will emit a compiler error: ``` C:\gitP\apache\arrow\cpp\src\arrow/json/object_writer.h(39): error C2039: 'string': is not a member of 'std' ``` ### What changes are included in this PR? The fix is simple and portable: include the necessary header. ### Are these changes tested? Nope, I'm totally YOLOing it. If it builds, it's good. (This will be tested in MSVC's internal "Real World Code" test infrastructure. Also, after VS 2022 17.11 ships, your existing build/test coverage will ensure that this keeps compiling.) ### Are there any user-facing changes? No. Authored-by: Stephan T. Lavavej Signed-off-by: Sutou Kouhei --- cpp/src/arrow/json/object_writer.h | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/src/arrow/json/object_writer.h b/cpp/src/arrow/json/object_writer.h index b15b09dbdacfc..cf1ce62194fb8 100644 --- a/cpp/src/arrow/json/object_writer.h +++ b/cpp/src/arrow/json/object_writer.h @@ -18,6 +18,7 @@ #pragma once #include +#include #include #include "arrow/util/visibility.h" From 290e606c4dd937cd34dbccd6f6801ff1ac1d8b9b Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 7 Jun 2024 11:34:22 +0900 Subject: [PATCH 79/93] GH-41652: [C++][CMake][Windows] Don't build needless object libraries (#41658) ### Rationale for this change * We don't need an object library for a shared library with `ARROW_BUILD_SHARED=OFF`. * We don't need an object library for a static library with `ARROW_BUILD_STATIC=OFF`. ### What changes are included in this PR? Don't build needless object libraries based on `ARROW_BUILD_SHARED`/`ARROW_BUILD_STATIC`. ### Are these changes tested? Yes. ### Are there any user-facing changes? No. * GitHub Issue: #41652 Authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- cpp/src/arrow/CMakeLists.txt | 37 +++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 150a304975cad..5bcd4625b3b67 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -200,22 +200,29 @@ function(arrow_add_object_library PREFIX) set(SOURCES ${ARGN}) string(TOLOWER "${PREFIX}" prefix) if(WIN32) - add_library(${prefix}_shared OBJECT ${SOURCES}) - add_library(${prefix}_static OBJECT ${SOURCES}) - set_target_properties(${prefix}_shared PROPERTIES POSITION_INDEPENDENT_CODE ON) - set_target_properties(${prefix}_static PROPERTIES POSITION_INDEPENDENT_CODE ON) - target_compile_definitions(${prefix}_shared PRIVATE ARROW_EXPORTING) - target_compile_definitions(${prefix}_static PRIVATE ARROW_STATIC) - target_compile_features(${prefix}_shared PRIVATE cxx_std_17) - target_compile_features(${prefix}_static PRIVATE cxx_std_17) - set(${PREFIX}_TARGET_SHARED - ${prefix}_shared - PARENT_SCOPE) - set(${PREFIX}_TARGET_STATIC - ${prefix}_static - PARENT_SCOPE) + set(targets) + if(ARROW_BUILD_SHARED) + add_library(${prefix}_shared OBJECT ${SOURCES}) + set_target_properties(${prefix}_shared PROPERTIES POSITION_INDEPENDENT_CODE ON) + target_compile_definitions(${prefix}_shared PRIVATE ARROW_EXPORTING) + target_compile_features(${prefix}_shared PRIVATE cxx_std_17) + set(${PREFIX}_TARGET_SHARED + ${prefix}_shared + PARENT_SCOPE) + list(APPEND targets ${prefix}_shared) + endif() + if(ARROW_BUILD_STATIC) + add_library(${prefix}_static OBJECT ${SOURCES}) + set_target_properties(${prefix}_static PROPERTIES POSITION_INDEPENDENT_CODE ON) + target_compile_definitions(${prefix}_static PRIVATE ARROW_STATIC) + target_compile_features(${prefix}_static PRIVATE cxx_std_17) + set(${PREFIX}_TARGET_STATIC + ${prefix}_static + PARENT_SCOPE) + list(APPEND targets ${prefix}_static) + endif() set(${PREFIX}_TARGETS - ${prefix}_shared ${prefix}_static + ${targets} PARENT_SCOPE) else() add_library(${prefix} OBJECT ${SOURCES}) From 01d2fa0d461869a07b2ffeee517beb8116bd0ce2 Mon Sep 17 00:00:00 2001 From: Laurent Goujon Date: Thu, 6 Jun 2024 23:19:06 -0700 Subject: [PATCH 80/93] GH-41307: [Java] Use org.apache:apache parent pom version 31 (#41772) Use/update Maven modules to `org.apache:apache:31` and clean up Maven modules to remove unnecessary configuration or outdated workarounds * Add `org.apache:apache:31` to `org.apache.arrow:arrow-bom` and `org.apache.arrow.maven.plugins:arrow-maven-plugins` to make them conformant with ASF standards * Update `org.apache.arrow:arrow-java-root` parent to `org.apache:parent:31` * Use `version.*` and other properties to override plugin versions defined by `org.apache:parent` * Move standalone plugin versions under pluginManagement at the top level * Cleanup redundant plugin version or configuration declaration * Update `maven-dependency-plugin` to 3.6.1 and add the required overrides when necessary * Update `maven-shade-plugin` to 3.5.1 (via `org.apache:parent`) - disable reduced dependency pom creation for non-terminal modules * Remove enforcer check for java and maven version (handled by `org.apache:parent`) * Remove unnecessary `mvnrepository` link comments * Remove `m2e.version` property check in profiles (only needed for errorprone plugin configuration which is incompatible with M2E) * Cleanup `argLine` overrides for surefire/failsafe plugins * Remove unnecessary `../pom.xml` `` directives * Remove source/target/encoding configuration properties for `maven-compiler-plugin`, `maven-javadoc-plugin` and `maven-resources-plugin` as it is handled by `org.apache:parent` and plugin themselves * Remove unnecessary copy of codegen templates in `arrow-vector` module * Remove unnecessary junit jupiter engine dependencies for surefire/failsafe plugins. * GitHub Issue: #41307 Lead-authored-by: Laurent Goujon Co-authored-by: Laurent Goujon Signed-off-by: David Li --- ci/scripts/java_full_build.sh | 10 +- dev/tasks/tasks.yml | 2 - java/adapter/avro/pom.xml | 9 - java/adapter/jdbc/pom.xml | 7 - java/adapter/orc/pom.xml | 17 ++ java/bom/pom.xml | 44 +++- java/c/pom.xml | 1 - java/flight/flight-core/pom.xml | 29 +-- java/flight/flight-integration-tests/pom.xml | 2 - java/flight/flight-sql-jdbc-core/pom.xml | 10 - java/flight/flight-sql-jdbc-driver/pom.xml | 1 - java/flight/flight-sql/pom.xml | 5 - java/format/pom.xml | 2 - java/gandiva/pom.xml | 19 +- .../module-info-compiler-maven-plugin/pom.xml | 28 +-- java/maven/pom.xml | 120 +++++----- java/memory/memory-core/pom.xml | 22 +- java/performance/pom.xml | 41 +--- java/pom.xml | 207 +++++++----------- java/tools/pom.xml | 22 +- java/vector/pom.xml | 94 +------- 21 files changed, 226 insertions(+), 466 deletions(-) diff --git a/ci/scripts/java_full_build.sh b/ci/scripts/java_full_build.sh index 2734f3e9dbec2..d914aa2d8472e 100755 --- a/ci/scripts/java_full_build.sh +++ b/ci/scripts/java_full_build.sh @@ -49,21 +49,13 @@ fi # build the entire project mvn clean \ install \ - assembly:single \ - source:jar \ - javadoc:jar \ -Papache-release \ -Parrow-c-data \ -Parrow-jni \ -Darrow.cpp.build.dir=$dist_dir \ - -Darrow.c.jni.dist.dir=$dist_dir \ - -DdescriptorId=source-release + -Darrow.c.jni.dist.dir=$dist_dir # copy all jar, zip and pom files to the distribution folder -find . \ - "(" -name "*-javadoc.jar" -o -name "*-sources.jar" ")" \ - -exec echo {} ";" \ - -exec cp {} $dist_dir ";" find ~/.m2/repository/org/apache/arrow \ "(" \ -name "*.jar" -o \ diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index d8e09ec2070bb..2d84751d0f363 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -747,7 +747,6 @@ tasks: - arrow-jdbc-{no_rc_snapshot_version}.pom - arrow-maven-plugins-{no_rc_snapshot_version}-cyclonedx.json - arrow-maven-plugins-{no_rc_snapshot_version}-cyclonedx.xml - - arrow-maven-plugins-{no_rc_snapshot_version}-src.zip - arrow-maven-plugins-{no_rc_snapshot_version}.pom - arrow-memory-core-{no_rc_snapshot_version}-cyclonedx.json - arrow-memory-core-{no_rc_snapshot_version}-cyclonedx.xml @@ -851,7 +850,6 @@ tasks: - module-info-compiler-maven-plugin-{no_rc_snapshot_version}-cyclonedx.xml - module-info-compiler-maven-plugin-{no_rc_snapshot_version}-javadoc.jar - module-info-compiler-maven-plugin-{no_rc_snapshot_version}-sources.jar - - module-info-compiler-maven-plugin-{no_rc_snapshot_version}-src.zip - module-info-compiler-maven-plugin-{no_rc_snapshot_version}.jar - module-info-compiler-maven-plugin-{no_rc_snapshot_version}.pom diff --git a/java/adapter/avro/pom.xml b/java/adapter/avro/pom.xml index 0046fcac62a22..f9bf29596796f 100644 --- a/java/adapter/avro/pom.xml +++ b/java/adapter/avro/pom.xml @@ -25,36 +25,27 @@ http://maven.apache.org - - org.apache.arrow arrow-memory-core - - org.apache.arrow arrow-memory-netty runtime - - org.apache.arrow arrow-vector - org.immutables value-annotations - org.apache.avro avro ${dep.avro.version} - diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index 17681538ac97e..2f2911dd9da95 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -26,20 +26,17 @@ - org.apache.arrow arrow-memory-core - org.apache.arrow arrow-memory-netty runtime - org.apache.arrow arrow-vector @@ -51,7 +48,6 @@ value-annotations - com.h2database h2 @@ -94,9 +90,6 @@ jdk11+ [11,] - - !m2e.version - diff --git a/java/adapter/orc/pom.xml b/java/adapter/orc/pom.xml index ca817510bf3e3..bc89c4698eecf 100644 --- a/java/adapter/orc/pom.xml +++ b/java/adapter/orc/pom.xml @@ -134,5 +134,22 @@ + + + org.apache.maven.plugins + maven-dependency-plugin + + + analyze + + + + org.apache.arrow:arrow-format + + + + + + diff --git a/java/bom/pom.xml b/java/bom/pom.xml index 12b9950ad80fc..77aed2d0f6a37 100644 --- a/java/bom/pom.xml +++ b/java/bom/pom.xml @@ -15,7 +15,7 @@ org.apache apache - 18 + 31 org.apache.arrow @@ -27,6 +27,19 @@ + + 1.8 + 1.8 + 3.12.0 + 3.2.5 + 0.16.1 + 3.7.1 + 3.12.1 + 3.6.1 + 3.2.4 + 3.2.2 + 3.6.3 + 3.5.0 @@ -138,11 +151,9 @@ ${project.version} - - @@ -156,12 +167,10 @@ org.apache.maven.plugins maven-project-info-reports-plugin - 3.5.0 org.apache.maven.plugins maven-site-plugin - 3.12.1 com.diffplug.spotless @@ -188,13 +197,34 @@ org.apache.maven.plugins maven-project-info-reports-plugin - 3.5.0 org.apache.maven.plugins maven-site-plugin - 3.12.1 + + + + apache-release + + + + org.apache.maven.plugins + maven-assembly-plugin + + + source-release-assembly + + + true + + + + + + + + diff --git a/java/c/pom.xml b/java/c/pom.xml index bfb233315a839..afb6e0cd8b890 100644 --- a/java/c/pom.xml +++ b/java/c/pom.xml @@ -83,5 +83,4 @@ - diff --git a/java/flight/flight-core/pom.xml b/java/flight/flight-core/pom.xml index b565572b383ab..f2070d4ff7cba 100644 --- a/java/flight/flight-core/pom.xml +++ b/java/flight/flight-core/pom.xml @@ -15,7 +15,6 @@ org.apache.arrow arrow-flight 17.0.0-SNAPSHOT - ../pom.xml flight-core @@ -151,13 +150,6 @@ org.apache.maven.plugins maven-shade-plugin - - 3.2.4 shade-main @@ -166,6 +158,7 @@ package + false true shaded @@ -192,6 +185,7 @@ package + false true shaded-ext @@ -244,7 +238,6 @@ org.apache.maven.plugins maven-dependency-plugin - 3.3.0 analyze @@ -264,7 +257,6 @@ org.codehaus.mojo build-helper-maven-plugin - 1.9.1 add-generated-sources-to-classpath @@ -282,7 +274,6 @@ maven-assembly-plugin - 3.7.1 jar-with-dependencies @@ -299,13 +290,6 @@ - - - kr.motd.maven - os-maven-plugin - 1.7.1 - - @@ -313,18 +297,14 @@ jdk11+ [11,] - - !m2e.version - org.apache.maven.plugins maven-surefire-plugin - - --add-opens=org.apache.arrow.flight.core/org.apache.arrow.flight.perf.impl=protobuf.java --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED - false + + --add-opens=org.apache.arrow.flight.core/org.apache.arrow.flight.perf.impl=protobuf.java --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED ${project.basedir}/../../../testing/data @@ -334,5 +314,4 @@ - diff --git a/java/flight/flight-integration-tests/pom.xml b/java/flight/flight-integration-tests/pom.xml index 74016d81e91e5..cd2c28ba8959f 100644 --- a/java/flight/flight-integration-tests/pom.xml +++ b/java/flight/flight-integration-tests/pom.xml @@ -15,7 +15,6 @@ org.apache.arrow arrow-flight 17.0.0-SNAPSHOT - ../pom.xml flight-integration-tests @@ -63,7 +62,6 @@ maven-assembly-plugin - 3.7.1 jar-with-dependencies diff --git a/java/flight/flight-sql-jdbc-core/pom.xml b/java/flight/flight-sql-jdbc-core/pom.xml index 459412e0f8d8b..50d7b2617a5a9 100644 --- a/java/flight/flight-sql-jdbc-core/pom.xml +++ b/java/flight/flight-sql-jdbc-core/pom.xml @@ -15,7 +15,6 @@ org.apache.arrow arrow-flight 17.0.0-SNAPSHOT - ../pom.xml flight-sql-jdbc-core @@ -47,20 +46,17 @@ - org.apache.arrow arrow-memory-core - org.apache.arrow arrow-memory-netty runtime - org.apache.arrow arrow-vector @@ -136,11 +132,6 @@ - - - src/main/resources - - maven-surefire-plugin @@ -154,7 +145,6 @@ org.codehaus.mojo properties-maven-plugin - 1.2.1 write-project-properties-to-file diff --git a/java/flight/flight-sql-jdbc-driver/pom.xml b/java/flight/flight-sql-jdbc-driver/pom.xml index b3afbe1defdba..4456270e7b347 100644 --- a/java/flight/flight-sql-jdbc-driver/pom.xml +++ b/java/flight/flight-sql-jdbc-driver/pom.xml @@ -15,7 +15,6 @@ org.apache.arrow arrow-flight 17.0.0-SNAPSHOT - ../pom.xml flight-sql-jdbc-driver diff --git a/java/flight/flight-sql/pom.xml b/java/flight/flight-sql/pom.xml index e6d703c673ad5..14fde34c3b4f3 100644 --- a/java/flight/flight-sql/pom.xml +++ b/java/flight/flight-sql/pom.xml @@ -15,7 +15,6 @@ org.apache.arrow arrow-flight 17.0.0-SNAPSHOT - ../pom.xml flight-sql @@ -119,9 +118,6 @@ jdk11+ [11,] - - !m2e.version - @@ -136,5 +132,4 @@ - diff --git a/java/format/pom.xml b/java/format/pom.xml index e9eded79de660..4483047e20960 100644 --- a/java/format/pom.xml +++ b/java/format/pom.xml @@ -31,7 +31,6 @@ - @@ -42,6 +41,5 @@ - diff --git a/java/gandiva/pom.xml b/java/gandiva/pom.xml index a87f26028ba86..1c17023e5c8ad 100644 --- a/java/gandiva/pom.xml +++ b/java/gandiva/pom.xml @@ -22,13 +22,12 @@ jar Arrow Gandiva Java wrappers around the native Gandiva SQL expression compiler. + - 1.8 - 1.8 - 3.25.1 true ../../../cpp/release-build + org.apache.arrow @@ -51,7 +50,6 @@ com.google.protobuf protobuf-java - ${protobuf.version} com.google.guava @@ -62,6 +60,7 @@ slf4j-api + @@ -88,14 +87,6 @@ - - - - kr.motd.maven - os-maven-plugin - 1.7.1 - - @@ -105,7 +96,6 @@ org.apache.maven.plugins maven-source-plugin - 2.2.1 attach-sources @@ -118,7 +108,6 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.6.3 attach-javadocs @@ -131,7 +120,6 @@ org.apache.maven.plugins maven-gpg-plugin - 3.2.4 sign-artifacts @@ -146,5 +134,4 @@ - diff --git a/java/maven/module-info-compiler-maven-plugin/pom.xml b/java/maven/module-info-compiler-maven-plugin/pom.xml index 57ba7933ea1c6..9f0cd7b1039dd 100644 --- a/java/maven/module-info-compiler-maven-plugin/pom.xml +++ b/java/maven/module-info-compiler-maven-plugin/pom.xml @@ -64,39 +64,14 @@ org.apache.maven.plugin-tools maven-plugin-annotations - 3.11.0 + ${maven.plugin.tools.version} provided - - - maven-clean-plugin - 3.3.2 - - - maven-plugin-plugin - 3.12.0 - - - maven-jar-plugin - 3.3.0 - - - maven-install-plugin - 3.1.2 - - - maven-deploy-plugin - 3.1.1 - - - maven-invoker-plugin - 3.1.0 - com.gradle develocity-maven-extension @@ -118,7 +93,6 @@ org.apache.maven.plugins maven-plugin-plugin - 3.12.0 true diff --git a/java/maven/pom.xml b/java/maven/pom.xml index 470e198caebc1..72140dd6570d0 100644 --- a/java/maven/pom.xml +++ b/java/maven/pom.xml @@ -15,6 +15,13 @@ Note: Do not inherit from the Arrow parent POM as plugins can be referenced during the parent POM, introducing circular dependencies. --> + + org.apache + apache + 31 + + + org.apache.arrow.maven.plugins arrow-maven-plugins 17.0.0-SNAPSHOT @@ -27,25 +34,38 @@ true + + 1.8 + 1.8 + 3.12.0 + 3.2.5 + 0.16.1 + 3.7.1 + 3.12.1 + 3.6.1 + 3.2.4 + 3.2.2 + 3.6.3 + 3.5.0 - org.apache.maven.plugins - maven-project-info-reports-plugin - 3.5.0 + com.diffplug.spotless + spotless-maven-plugin + 2.30.0 - org.apache.maven.plugins - maven-site-plugin - 3.12.1 + pl.project13.maven + git-commit-id-plugin + 4.0.5 - com.diffplug.spotless - spotless-maven-plugin - 2.30.0 + org.cyclonedx + cyclonedx-maven-plugin + 2.7.11 @@ -119,11 +139,6 @@ **/logback.xml - true - - true - true - org.apache.arrow ${username} @@ -143,43 +158,17 @@ - - org.apache.maven.plugins - maven-resources-plugin - - UTF-8 - - org.apache.maven.plugins maven-compiler-plugin - UTF-8 - 1.8 - 1.8 2048m - false true maven-enforcer-plugin - - validate_java_and_maven_version - - enforce - - verify - false - - - - [3.3.0,4) - - - - avoid_bad_dependencies @@ -205,8 +194,6 @@ pl.project13.maven git-commit-id-plugin - 4.0.5 - dd.MM.yyyy '@' HH:mm:ss z false @@ -248,7 +235,6 @@ org.apache.maven.plugins maven-checkstyle-plugin - 3.1.0 ../dev/checkstyle/checkstyle.xml ../dev/checkstyle/checkstyle.license @@ -288,7 +274,6 @@ org.cyclonedx cyclonedx-maven-plugin - 2.7.11 @@ -298,28 +283,6 @@ - - - org.apache.maven.plugins - maven-assembly-plugin - - - src - - - - - - single - - package - - - - org.apache.maven.plugins maven-project-info-reports-plugin @@ -353,13 +316,34 @@ org.apache.maven.plugins maven-project-info-reports-plugin - 3.5.0 org.apache.maven.plugins maven-site-plugin - 3.12.1 + + + + apache-release + + + + org.apache.maven.plugins + maven-assembly-plugin + + + source-release-assembly + + + true + + + + + + + + diff --git a/java/memory/memory-core/pom.xml b/java/memory/memory-core/pom.xml index 1e29ccf8ab9db..783a13a6fb0ad 100644 --- a/java/memory/memory-core/pom.xml +++ b/java/memory/memory-core/pom.xml @@ -61,9 +61,6 @@ jdk11+ [11,] - - !m2e.version - @@ -92,7 +89,6 @@ org.apache.maven.plugins maven-surefire-plugin - opens-tests @@ -101,12 +97,9 @@ test - - -Dfoo=bar - - - **/TestArrowBuf.java - + + + **/TestOpens.java @@ -129,9 +122,6 @@ org.apache.maven.plugins maven-compiler-plugin - 8 - 8 - UTF-8 -Xmaxerrs @@ -150,12 +140,6 @@ ${checker.framework.version} - - - org.immutables.value.internal.$processor$.$Processor - - org.checkerframework.checker.nullness.NullnessChecker - diff --git a/java/performance/pom.xml b/java/performance/pom.xml index f01e8d9a4e0e4..07ca8d1e61d48 100644 --- a/java/performance/pom.xml +++ b/java/performance/pom.xml @@ -22,9 +22,7 @@ JMH Performance benchmarks for other Arrow libraries. - UTF-8 1.37 - 1.8 benchmarks true .* @@ -83,42 +81,6 @@ - - - - maven-clean-plugin - 3.3.2 - - - maven-deploy-plugin - 3.1.1 - - - maven-install-plugin - 3.1.2 - - - maven-jar-plugin - 3.3.0 - - - maven-javadoc-plugin - 3.6.3 - - - maven-resources-plugin - 3.3.1 - - - maven-source-plugin - 2.2.1 - - - maven-surefire-plugin - 3.2.5 - - - org.apache.maven.plugins @@ -144,6 +106,7 @@ package ${uberjar.name} + false org.openjdk.jmh.Main @@ -166,7 +129,6 @@ org.codehaus.mojo exec-maven-plugin - 3.2.0 ${skip.perf.benchmarks} test @@ -203,5 +165,4 @@ - diff --git a/java/pom.xml b/java/pom.xml index 0e9b7f0e25a34..9624444cf422d 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -15,7 +15,7 @@ org.apache apache - 18 + 31 org.apache.arrow @@ -85,7 +85,7 @@ 33.2.1-jre 4.1.108.Final 1.63.0 - 3.23.1 + 3.25.1 2.17.0 3.4.0 23.5.26 @@ -95,10 +95,28 @@ true 9+181-r4173-1 2.28.0 - 3.12.1 5.11.0 5.2.0 3.43.0 + none + -Xdoclint:none + + 1.8 + 1.8 + 3.12.0 + 3.2.5 + 0.16.1 + 3.7.1 + 3.12.1 + 3.6.1 + 3.2.4 + + 3.2.2 + 3.6.3 + 3.5.0 @@ -269,40 +287,16 @@ 8.3.0 test - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - 3.1.2 - - - org.apache.rat - apache-rat-plugin - 0.16.1 - - - org.apache.maven.plugins - maven-resources-plugin - 3.3.1 - org.apache.maven.plugins maven-compiler-plugin - ${maven-compiler-plugin.version} + true **/module-info.java **/module-info.java false @@ -315,18 +309,8 @@ - - maven-enforcer-plugin - 3.4.1 - - - org.apache.maven.plugins - maven-shade-plugin - 3.5.1 - maven-surefire-plugin - 3.2.5 true true @@ -341,22 +325,9 @@ 1048576 - - - org.junit.jupiter - junit-jupiter-engine - ${dep.junit.jupiter.version} - - - org.apache.maven.surefire - surefire-junit-platform - 3.2.5 - - maven-failsafe-plugin - 3.2.5 ${project.build.directory} @@ -445,6 +416,22 @@ + + + org.apache.drill.tools + drill-fmpp-maven-plugin + [1.0,) + + generate + + + + + false + true + + + @@ -452,9 +439,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.6.3 - 8 **/module-info.java @@ -465,16 +450,6 @@ module-info-compiler-maven-plugin ${project.version} - - org.apache.maven.plugins - maven-project-info-reports-plugin - 3.5.0 - - - org.apache.maven.plugins - maven-site-plugin - 3.12.1 - com.gradle develocity-maven-extension @@ -522,6 +497,36 @@ spotless-maven-plugin 2.30.0 + + org.codehaus.mojo + build-helper-maven-plugin + 1.9.1 + + + org.codehaus.mojo + properties-maven-plugin + 1.2.1 + + + org.codehaus.mojo + exec-maven-plugin + 3.2.0 + + + pl.project13.maven + git-commit-id-plugin + 4.0.5 + + + org.cyclonedx + cyclonedx-maven-plugin + 2.7.11 + + + org.apache.drill.tools + drill-fmpp-maven-plugin + 1.21.1 + @@ -595,11 +600,6 @@ **/logback.xml - true - - true - true - org.apache.arrow ${username} @@ -619,42 +619,17 @@ - - org.apache.maven.plugins - maven-resources-plugin - - UTF-8 - - org.apache.maven.plugins maven-compiler-plugin - 1.8 - 1.8 2048m - false true maven-enforcer-plugin - - validate_java_and_maven_version - - enforce - - verify - false - - - - [3.3.0,4) - - - - avoid_bad_dependencies @@ -683,8 +658,6 @@ pl.project13.maven git-commit-id-plugin - 4.0.5 - dd.MM.yyyy '@' HH:mm:ss z false @@ -726,7 +699,6 @@ org.apache.maven.plugins maven-checkstyle-plugin - 3.1.0 **/module-info.java dev/checkstyle/checkstyle.xml @@ -789,7 +761,6 @@ org.cyclonedx cyclonedx-maven-plugin - 2.7.11 @@ -820,12 +791,10 @@ org.apache.maven.plugins maven-project-info-reports-plugin - 3.5.0 org.apache.maven.plugins maven-site-plugin - 3.12.1 com.diffplug.spotless @@ -860,7 +829,6 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.6.3 **/module-info.java @@ -888,28 +856,15 @@ org.apache.maven.plugins maven-project-info-reports-plugin - 3.5.0 org.apache.maven.plugins maven-site-plugin - 3.12.1 - - java-nodoclint - - [1.8,) - - - none - -Xdoclint:none - - - arrow-c-data @@ -960,7 +915,6 @@ org.apache.maven.plugins maven-compiler-plugin - true -XDcompilePolicy=simple -Xplugin:ErrorProne @@ -1000,9 +954,6 @@ org.apache.maven.plugins maven-compiler-plugin - 8 - 8 - UTF-8 -XDcompilePolicy=simple -Xplugin:ErrorProne -XepExcludedPaths:.*/(target/generated-source|format/src/main/java/org/apache/arrow/flatbuf)/.* @@ -1026,6 +977,16 @@ + + + + + jdk11+ + + [11,] + + + org.apache.maven.plugins maven-surefire-plugin @@ -1033,6 +994,13 @@ --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED + + org.apache.maven.plugins + maven-failsafe-plugin + + --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED + + @@ -1073,7 +1041,6 @@ org.jacoco jacoco-maven-plugin - 0.8.11 @@ -1119,7 +1086,6 @@ org.codehaus.mojo exec-maven-plugin - 3.2.0 cdata-cmake @@ -1176,7 +1142,6 @@ org.codehaus.mojo exec-maven-plugin - 3.2.0 jni-cpp-cmake @@ -1283,7 +1248,6 @@ org.codehaus.mojo exec-maven-plugin - 3.2.0 jni-cpp-cmake @@ -1373,5 +1337,4 @@ - diff --git a/java/tools/pom.xml b/java/tools/pom.xml index 5d9db75e525bd..53dcd51771054 100644 --- a/java/tools/pom.xml +++ b/java/tools/pom.xml @@ -54,6 +54,11 @@ 1.3.14 test + com.fasterxml.jackson.core jackson-core @@ -85,7 +90,6 @@ maven-assembly-plugin - 3.7.1 jar-with-dependencies @@ -101,7 +105,21 @@ + + org.apache.maven.plugins + maven-dependency-plugin + + + analyze + verify + + + com.fasterxml.jackson.core:* + + + + + - diff --git a/java/vector/pom.xml b/java/vector/pom.xml index c39504df2b207..6ff869ee21aff 100644 --- a/java/vector/pom.xml +++ b/java/vector/pom.xml @@ -76,64 +76,7 @@ - - - - true - - - false - - apache - apache - https://repo.maven.apache.org/maven2/ - - - - - - - codegen - - ${basedir}/src/main/codegen - - - - - - - org.eclipse.m2e - lifecycle-mapping - 1.0.0 - - - - - - org.apache.drill.tools - drill-fmpp-maven-plugin - [1.0,) - - generate - - - - - false - true - - - - - - - - - - org.apache.maven.plugins @@ -163,33 +106,10 @@ - - maven-resources-plugin - - - - copy-fmpp-resources - - copy-resources - - initialize - - ${project.build.directory}/codegen - - - src/main/codegen - false - - - - - - org.apache.drill.tools drill-fmpp-maven-plugin - 1.21.1 generate-fmpp @@ -200,7 +120,7 @@ src/main/codegen/config.fmpp ${project.build.directory}/generated-sources/fmpp - ${project.build.directory}/codegen/templates + src/main/codegen/templates @@ -208,13 +128,6 @@ org.apache.maven.plugins maven-shade-plugin - - 3.2.4 @@ -228,10 +141,9 @@ com.google.flatbuffers:* + false true shade-format-flatbuffers - true - true com.google.flatbuffers @@ -243,7 +155,6 @@ - @@ -276,5 +187,4 @@ - From b51e997df7dcde843befffed2d63d6a8e741beef Mon Sep 17 00:00:00 2001 From: Haocheng Liu <30446009+HaochengLIU@users.noreply.github.com> Date: Fri, 7 Jun 2024 04:03:15 -0400 Subject: [PATCH 81/93] GH-41960: Expose new S3 option check_directory_existence_before_creation (#41972) ### Rationale for this change Expose new S3 option `check_directory_existence_before_creation` from GH-41493 ### What changes are included in this PR? Expose new S3 option `check_directory_existence_before_creation` from GH-41493 ### Are these changes tested? yes ### Are there any user-facing changes? Yes. Python function documentation is updated. * GitHub Issue: #41960 Lead-authored-by: Haocheng Liu Co-authored-by: Joris Van den Bossche Signed-off-by: Joris Van den Bossche --- python/pyarrow/_s3fs.pyx | 20 ++++++++++++++++---- python/pyarrow/includes/libarrow_fs.pxd | 1 + python/pyarrow/tests/test_fs.py | 5 +++++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/python/pyarrow/_s3fs.pyx b/python/pyarrow/_s3fs.pyx index f5bab99a49f7a..ba6603322838d 100644 --- a/python/pyarrow/_s3fs.pyx +++ b/python/pyarrow/_s3fs.pyx @@ -185,7 +185,7 @@ cdef class S3FileSystem(FileSystem): session_token : str, default None AWS Session Token. An optional session token, required if access_key and secret_key are temporary credentials from STS. - anonymous : boolean, default False + anonymous : bool, default False Whether to connect anonymously if access_key and secret_key are None. If true, will not attempt to look up credentials using standard AWS configuration methods. @@ -217,7 +217,7 @@ cdef class S3FileSystem(FileSystem): S3 connection transport scheme. endpoint_override : str, default None Override region with a connect string such as "localhost:9000" - background_writes : boolean, default True + background_writes : bool, default True Whether file writes will be issued in the background, without blocking. default_metadata : mapping or pyarrow.KeyValueMetadata, default None @@ -237,11 +237,20 @@ cdef class S3FileSystem(FileSystem): 'port': 8020, 'username': 'username', 'password': 'password'}) allow_bucket_creation : bool, default False - Whether to allow CreateDir at the bucket-level. This option may also be + Whether to allow directory creation at the bucket-level. This option may also be passed in a URI query parameter. allow_bucket_deletion : bool, default False - Whether to allow DeleteDir at the bucket-level. This option may also be + Whether to allow directory deletion at the bucket-level. This option may also be passed in a URI query parameter. + check_directory_existence_before_creation : bool, default false + Whether to check the directory existence before creating it. + If false, when creating a directory the code will not check if it already + exists or not. It's an optimization to try directory creation and catch the error, + rather than issue two dependent I/O calls. + If true, when creating a directory the code will only create the directory when necessary + at the cost of extra I/O calls. This can be used for key/value cloud storage which has + a hard rate limit to number of object mutation operations or scenerios such as + the directories already exist and you do not have creation access. retry_strategy : S3RetryStrategy, default AwsStandardS3RetryStrategy(max_attempts=3) The retry strategy to use with S3; fail after max_attempts. Available strategies are AwsStandardS3RetryStrategy, AwsDefaultS3RetryStrategy. @@ -273,6 +282,7 @@ cdef class S3FileSystem(FileSystem): role_arn=None, session_name=None, external_id=None, load_frequency=900, proxy_options=None, allow_bucket_creation=False, allow_bucket_deletion=False, + check_directory_existence_before_creation=False, retry_strategy: S3RetryStrategy = AwsStandardS3RetryStrategy( max_attempts=3), force_virtual_addressing=False): @@ -387,6 +397,7 @@ cdef class S3FileSystem(FileSystem): options.value().allow_bucket_creation = allow_bucket_creation options.value().allow_bucket_deletion = allow_bucket_deletion + options.value().check_directory_existence_before_creation = check_directory_existence_before_creation options.value().force_virtual_addressing = force_virtual_addressing if isinstance(retry_strategy, AwsStandardS3RetryStrategy): @@ -447,6 +458,7 @@ cdef class S3FileSystem(FileSystem): background_writes=opts.background_writes, allow_bucket_creation=opts.allow_bucket_creation, allow_bucket_deletion=opts.allow_bucket_deletion, + check_directory_existence_before_creation=opts.check_directory_existence_before_creation, default_metadata=pyarrow_wrap_metadata(opts.default_metadata), proxy_options={'scheme': frombytes(opts.proxy_options.scheme), 'host': frombytes(opts.proxy_options.host), diff --git a/python/pyarrow/includes/libarrow_fs.pxd b/python/pyarrow/includes/libarrow_fs.pxd index f1f2985f65394..cc260b80c7779 100644 --- a/python/pyarrow/includes/libarrow_fs.pxd +++ b/python/pyarrow/includes/libarrow_fs.pxd @@ -157,6 +157,7 @@ cdef extern from "arrow/filesystem/api.h" namespace "arrow::fs" nogil: c_bool background_writes c_bool allow_bucket_creation c_bool allow_bucket_deletion + c_bool check_directory_existence_before_creation c_bool force_virtual_addressing shared_ptr[const CKeyValueMetadata] default_metadata c_string role_arn diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py index 845f1eccecc72..58380f1652558 100644 --- a/python/pyarrow/tests/test_fs.py +++ b/python/pyarrow/tests/test_fs.py @@ -1226,6 +1226,11 @@ def test_s3_options(pickle_module): assert isinstance(fs, S3FileSystem) assert pickle_module.loads(pickle_module.dumps(fs)) == fs + fs = S3FileSystem(allow_bucket_creation=True, allow_bucket_deletion=True, + check_directory_existence_before_creation=True) + assert isinstance(fs, S3FileSystem) + assert pickle_module.loads(pickle_module.dumps(fs)) == fs + fs = S3FileSystem(request_timeout=0.5, connect_timeout=0.25) assert isinstance(fs, S3FileSystem) assert pickle_module.loads(pickle_module.dumps(fs)) == fs From 1dde3995238d4a771c9525e1e5189c1db4a8a95a Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 7 Jun 2024 17:12:53 +0900 Subject: [PATCH 82/93] GH-42017: [CI][Python][C++] Fix utf8proc detection for wheel on Windows (#42022) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change utf8proc in vcpkg provides CMake package. If we use it, we don't need to care about static library name (`utf8proc.lib` or `utf8proc_static.lib`). ### What changes are included in this PR? Use `unofficial-utf8proc` CMake package with vcpkg. ### Are these changes tested? Yes. ### Are there any user-facing changes? No. * GitHub Issue: #42017 Authored-by: Sutou Kouhei Signed-off-by: Raúl Cumplido --- cpp/cmake_modules/Findutf8proc.cmake | 2 +- cpp/cmake_modules/ThirdpartyToolchain.cmake | 12 +++++++----- cpp/cmake_modules/Usevcpkg.cmake | 3 --- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/cpp/cmake_modules/Findutf8proc.cmake b/cpp/cmake_modules/Findutf8proc.cmake index e347414090549..9721f76f0631b 100644 --- a/cpp/cmake_modules/Findutf8proc.cmake +++ b/cpp/cmake_modules/Findutf8proc.cmake @@ -19,7 +19,7 @@ if(utf8proc_FOUND) return() endif() -if(ARROW_PACKAGE_KIND STREQUAL "vcpkg") +if(ARROW_PACKAGE_KIND STREQUAL "vcpkg" OR VCPKG_TOOLCHAIN) set(find_package_args "") if(utf8proc_FIND_VERSION) list(APPEND find_package_args ${utf8proc_FIND_VERSION}) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index f102c7bb81683..3c58ba649c4dd 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2819,11 +2819,13 @@ macro(build_utf8proc) endmacro() if(ARROW_WITH_UTF8PROC) - resolve_dependency(utf8proc - PC_PACKAGE_NAMES - libutf8proc - REQUIRED_VERSION - "2.2.0") + set(utf8proc_resolve_dependency_args utf8proc PC_PACKAGE_NAMES libutf8proc) + if(NOT VCPKG_TOOLCHAIN) + # utf8proc in vcpkg doesn't provide version information: + # https://github.com/microsoft/vcpkg/issues/39176 + list(APPEND utf8proc_resolve_dependency_args REQUIRED_VERSION "2.2.0") + endif() + resolve_dependency(${utf8proc_resolve_dependency_args}) endif() macro(build_cares) diff --git a/cpp/cmake_modules/Usevcpkg.cmake b/cpp/cmake_modules/Usevcpkg.cmake index 37a732f4b85a0..b6192468da342 100644 --- a/cpp/cmake_modules/Usevcpkg.cmake +++ b/cpp/cmake_modules/Usevcpkg.cmake @@ -237,9 +237,6 @@ set(LZ4_ROOT CACHE STRING "") if(CMAKE_HOST_WIN32) - set(utf8proc_MSVC_STATIC_LIB_SUFFIX - "" - CACHE STRING "") set(LZ4_MSVC_LIB_PREFIX "" CACHE STRING "") From a045770b94972bb4063bde13cb95f1c5b5c8bbe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 7 Jun 2024 10:16:50 +0200 Subject: [PATCH 83/93] GH-42006: [CI][Python] Use pip install -e instead of setup.py build_ext --inplace for installing pyarrow on verification script (#42007) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change Due to https://github.com/apache/arrow/issues/37929 we require a higher version of setuptools and setuptools_scm to be installed otherwise the job fails with setuptools_scm failing with ` TypeError: Configuration.__init__() got an unexpected keyword argument 'version_file'` ### What changes are included in this PR? Remove the dependencies for the environment and let installation handle those using pip install -e instead of setup.py build_ext --inplace for installing pyarrow on verification script ### Are these changes tested? Via Archery ### Are there any user-facing changes? No * GitHub Issue: #42006 Lead-authored-by: Raúl Cumplido Co-authored-by: Joris Van den Bossche Signed-off-by: Raúl Cumplido --- dev/release/verify-release-candidate.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 3ed871bd5305b..fcaaa423a4c75 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -756,7 +756,7 @@ test_python() { show_header "Build and test Python libraries" # Build and test Python - maybe_setup_virtualenv "cython>=0.29.31" numpy "setuptools_scm<8.0.0" setuptools + maybe_setup_virtualenv maybe_setup_conda --file ci/conda_env_python.txt if [ "${USE_CONDA}" -gt 0 ]; then @@ -788,7 +788,7 @@ test_python() { pushd python # Build pyarrow - python setup.py build_ext --inplace + python -m pip install -e . # Check mandatory and optional imports python -c " From fe4d04f081e55ca2de7b1b67b10ad7dca96cfd9e Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Fri, 7 Jun 2024 22:26:34 +0900 Subject: [PATCH 84/93] GH-42002: [Java] Update Unit Tests for Vector Module (#42019) ### Rationale for this change Update package from JUnit 4(`org.junit`) to JUnit 5(`org.junit.jupiter`). ### What changes are included in this PR? - [x] Replacing `org.junit` with `org.junit.jupiter.api`. - [x] Updating `Assertions.assertXXX` to `assertXXX` using static imports. - [x] Updating annotations such as `@ Before`, `@ BeforeClass`, `@ After`, `@ AfterClass`. - `@ Before` -> `@ BeforeEach` - `@ BeforeClass` -> `@ BeforeAll` - `@ After` -> `@ AfterEach` - `@ AfterClass` -> `@ AfterAll` - `@ Test` -> `@ Test` with `org.junit.jupiter` - [x] Removing unused `@ Rule` Annotation - [x] Updating `Parameterized` test - [x] Doing self review ### Are these changes tested? Yes, existing tests have passed. ### Are there any user-facing changes? No. * GitHub Issue: #42002 Authored-by: Hyunseok Seo Signed-off-by: David Li --- .../arrow/vector/ITTestLargeVector.java | 8 +- .../apache/arrow/vector/TestBitVector.java | 73 +++--- .../arrow/vector/TestBitVectorHelper.java | 6 +- .../vector/TestBufferOwnershipTransfer.java | 8 +- .../org/apache/arrow/vector/TestCopyFrom.java | 85 +++--- .../arrow/vector/TestDecimal256Vector.java | 18 +- .../arrow/vector/TestDecimalVector.java | 20 +- .../arrow/vector/TestDenseUnionVector.java | 25 +- .../arrow/vector/TestDictionaryVector.java | 28 +- .../arrow/vector/TestDurationVector.java | 16 +- .../vector/TestFixedSizeBinaryVector.java | 16 +- .../arrow/vector/TestFixedSizeListVector.java | 120 ++++----- .../TestIntervalMonthDayNanoVector.java | 13 +- .../arrow/vector/TestIntervalYearVector.java | 14 +- .../arrow/vector/TestLargeListVector.java | 35 ++- .../vector/TestLargeVarBinaryVector.java | 18 +- .../arrow/vector/TestLargeVarCharVector.java | 92 +++---- .../apache/arrow/vector/TestListVector.java | 35 ++- .../arrow/vector/TestNullCheckingForGet.java | 10 +- .../vector/TestOutOfMemoryForValueVector.java | 52 ++-- ...TestOversizedAllocationForValueVector.java | 155 +++++------ .../arrow/vector/TestPeriodDuration.java | 6 +- .../apache/arrow/vector/TestStructVector.java | 23 +- .../apache/arrow/vector/TestUnionVector.java | 25 +- .../arrow/vector/TestVarCharListVector.java | 17 +- .../apache/arrow/vector/TestVectorAlloc.java | 14 +- .../arrow/vector/TestVectorReAlloc.java | 70 ++--- .../apache/arrow/vector/TestVectorReset.java | 14 +- .../arrow/vector/TestVectorSchemaRoot.java | 43 +-- .../arrow/vector/TestVectorUnloadLoad.java | 35 ++- .../vector/compare/TestTypeEqualsVisitor.java | 14 +- .../complex/TestDenseUnionBufferSize.java | 3 +- .../complex/impl/TestComplexCopier.java | 13 +- .../complex/impl/TestPromotableWriter.java | 42 +-- .../complex/writer/TestComplexWriter.java | 244 +++++++++--------- .../complex/writer/TestSimpleWriter.java | 38 +-- .../apache/arrow/vector/ipc/BaseFileTest.java | 209 +++++++-------- .../ipc/ITTestIPCWithLargeArrowBuffers.java | 10 +- .../vector/ipc/MessageSerializerTest.java | 14 +- .../arrow/vector/ipc/TestArrowFile.java | 6 +- .../arrow/vector/ipc/TestArrowFooter.java | 4 +- .../vector/ipc/TestArrowReaderWriter.java | 37 ++- .../arrow/vector/ipc/TestArrowStream.java | 15 +- .../arrow/vector/ipc/TestArrowStreamPipe.java | 20 +- .../apache/arrow/vector/ipc/TestJSONFile.java | 11 +- .../arrow/vector/ipc/TestRoundTrip.java | 158 +++++++----- .../ipc/TestUIntDictionaryRoundTrip.java | 88 +++---- .../message/TestMessageMetadataResult.java | 4 +- .../apache/arrow/vector/pojo/TestConvert.java | 6 +- .../testing/TestValueVectorPopulator.java | 12 +- .../testing/ValueVectorDataPopulator.java | 2 +- .../vector/types/pojo/TestExtensionType.java | 67 +++-- .../arrow/vector/types/pojo/TestField.java | 8 +- .../arrow/vector/types/pojo/TestSchema.java | 10 +- .../arrow/vector/util/DecimalUtilityTest.java | 19 +- .../vector/util/TestDataSizeRoundingUtil.java | 4 +- .../TestElementAddressableVectorIterator.java | 14 +- .../arrow/vector/util/TestMapWithOrdinal.java | 12 +- .../vector/util/TestMultiMapWithOrdinal.java | 43 +-- .../vector/util/TestReusableByteArray.java | 20 +- .../arrow/vector/util/TestSchemaUtil.java | 4 +- .../arrow/vector/util/TestValidator.java | 6 +- .../arrow/vector/util/TestVectorAppender.java | 14 +- .../vector/util/TestVectorBatchAppender.java | 12 +- .../util/TestVectorSchemaRootAppender.java | 12 +- .../vector/validate/TestValidateVector.java | 12 +- .../validate/TestValidateVectorFull.java | 12 +- .../TestValidateVectorSchemaRoot.java | 12 +- .../TestValidateVectorTypeVisitor.java | 10 +- 69 files changed, 1191 insertions(+), 1144 deletions(-) diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ITTestLargeVector.java b/java/vector/src/test/java/org/apache/arrow/vector/ITTestLargeVector.java index 8596399e7e08c..b65e6fd36c158 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ITTestLargeVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ITTestLargeVector.java @@ -17,9 +17,9 @@ package org.apache.arrow.vector; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.math.BigDecimal; import java.nio.charset.StandardCharsets; @@ -28,7 +28,7 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.holders.NullableDecimalHolder; -import org.junit.Test; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java index 075a05c04b641..cebd70fcc5a71 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVector.java @@ -17,10 +17,10 @@ package org.apache.arrow.vector; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.stream.IntStream; @@ -29,22 +29,21 @@ import org.apache.arrow.memory.util.hash.MurmurHasher; import org.apache.arrow.vector.testing.ValueVectorDataPopulator; import org.apache.arrow.vector.util.TransferPair; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestBitVector { private static final String EMPTY_SCHEMA_PATH = ""; private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new RootAllocator(Long.MAX_VALUE); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } @@ -124,8 +123,8 @@ public void testSplitAndTransfer() throws Exception { for (int i = 0; i < length; i++) { int actual = toVector.get(i); int expected = sourceVector.get(start + i); - assertEquals("different data values not expected --> sourceVector index: " + (start + i) + - " toVector index: " + i, expected, actual); + assertEquals(expected, actual, + "different data values not expected --> sourceVector index: " + (start + i) + " toVector index: " + i); } } } @@ -167,8 +166,8 @@ public void testSplitAndTransfer1() throws Exception { for (int i = 0; i < length; i++) { int actual = toVector.get(i); int expected = sourceVector.get(start + i); - assertEquals("different data values not expected --> sourceVector index: " + (start + i) + - " toVector index: " + i, expected, actual); + assertEquals(expected, actual, + "different data values not expected --> sourceVector index: " + (start + i) + " toVector index: " + i); } } } @@ -218,8 +217,8 @@ public void testSplitAndTransfer2() throws Exception { for (int i = 0; i < length; i++) { int actual = toVector.get(i); int expected = sourceVector.get(start + i); - assertEquals("different data values not expected --> sourceVector index: " + (start + i) + - " toVector index: " + i, expected, actual); + assertEquals(expected, actual, + "different data values not expected --> sourceVector index: " + (start + i) + " toVector index: " + i); } } } @@ -241,9 +240,9 @@ public void testReallocAfterVectorTransfer1() { for (int i = 0; i < valueCapacity; i++) { if ((i & 1) == 1) { - assertEquals("unexpected cleared bit at index: " + i, 1, vector.get(i)); + assertEquals(1, vector.get(i), "unexpected cleared bit at index: " + i); } else { - assertTrue("unexpected set bit at index: " + i, vector.isNull(i)); + assertTrue(vector.isNull(i), "unexpected set bit at index: " + i); } } @@ -259,9 +258,9 @@ public void testReallocAfterVectorTransfer1() { for (int i = 0; i < valueCapacity * 2; i++) { if (((i & 1) == 1) || (i == valueCapacity)) { - assertEquals("unexpected cleared bit at index: " + i, 1, vector.get(i)); + assertEquals(1, vector.get(i), "unexpected cleared bit at index: " + i); } else { - assertTrue("unexpected set bit at index: " + i, vector.isNull(i)); + assertTrue(vector.isNull(i), "unexpected set bit at index: " + i); } } @@ -277,9 +276,9 @@ public void testReallocAfterVectorTransfer1() { for (int i = 0; i < valueCapacity * 4; i++) { if (((i & 1) == 1) || (i == valueCapacity) || (i == valueCapacity * 2)) { - assertEquals("unexpected cleared bit at index: " + i, 1, vector.get(i)); + assertEquals(1, vector.get(i), "unexpected cleared bit at index: " + i); } else { - assertTrue("unexpected set bit at index: " + i, vector.isNull(i)); + assertTrue(vector.isNull(i), "unexpected set bit at index: " + i); } } @@ -297,12 +296,12 @@ public void testReallocAfterVectorTransfer1() { if (i <= valueCapacity * 4) { if (((i & 1) == 1) || (i == valueCapacity) || (i == valueCapacity * 2) || (i == valueCapacity * 4)) { - assertEquals("unexpected cleared bit at index: " + i, 1, toVector.get(i)); + assertEquals(1, toVector.get(i), "unexpected cleared bit at index: " + i); } else { - assertTrue("unexpected set bit at index: " + i, toVector.isNull(i)); + assertTrue(toVector.isNull(i), "unexpected set bit at index: " + i); } } else { - assertTrue("unexpected set bit at index: " + i, toVector.isNull(i)); + assertTrue(toVector.isNull(i), "unexpected set bit at index: " + i); } } @@ -325,9 +324,9 @@ public void testReallocAfterVectorTransfer2() { for (int i = 0; i < valueCapacity; i++) { if ((i & 1) == 1) { - assertFalse("unexpected cleared bit at index: " + i, vector.isNull(i)); + assertFalse(vector.isNull(i), "unexpected cleared bit at index: " + i); } else { - assertTrue("unexpected set bit at index: " + i, vector.isNull(i)); + assertTrue(vector.isNull(i), "unexpected set bit at index: " + i); } } @@ -343,9 +342,9 @@ public void testReallocAfterVectorTransfer2() { for (int i = 0; i < valueCapacity * 2; i++) { if (((i & 1) == 1) || (i == valueCapacity)) { - assertFalse("unexpected cleared bit at index: " + i, vector.isNull(i)); + assertFalse(vector.isNull(i), "unexpected cleared bit at index: " + i); } else { - assertTrue("unexpected set bit at index: " + i, vector.isNull(i)); + assertTrue(vector.isNull(i), "unexpected set bit at index: " + i); } } @@ -361,9 +360,9 @@ public void testReallocAfterVectorTransfer2() { for (int i = 0; i < valueCapacity * 4; i++) { if (((i & 1) == 1) || (i == valueCapacity) || (i == valueCapacity * 2)) { - assertFalse("unexpected cleared bit at index: " + i, vector.isNull(i)); + assertFalse(vector.isNull(i), "unexpected cleared bit at index: " + i); } else { - assertTrue("unexpected set bit at index: " + i, vector.isNull(i)); + assertTrue(vector.isNull(i), "unexpected set bit at index: " + i); } } @@ -381,12 +380,12 @@ public void testReallocAfterVectorTransfer2() { if (i <= valueCapacity * 4) { if (((i & 1) == 1) || (i == valueCapacity) || (i == valueCapacity * 2) || (i == valueCapacity * 4)) { - assertFalse("unexpected cleared bit at index: " + i, toVector.isNull(i)); + assertFalse(toVector.isNull(i), "unexpected cleared bit at index: " + i); } else { - assertTrue("unexpected set bit at index: " + i, toVector.isNull(i)); + assertTrue(toVector.isNull(i), "unexpected set bit at index: " + i); } } else { - assertTrue("unexpected set bit at index: " + i, toVector.isNull(i)); + assertTrue(toVector.isNull(i), "unexpected set bit at index: " + i); } } @@ -500,13 +499,13 @@ private void validateRange(int length, int start, int count) { bitVector.allocateNew(length); bitVector.setRangeToOne(start, count); for (int i = 0; i < start; i++) { - Assert.assertTrue(desc + i, bitVector.isNull(i)); + assertTrue(bitVector.isNull(i), desc + i); } for (int i = start; i < start + count; i++) { - Assert.assertEquals(desc + i, 1, bitVector.get(i)); + assertEquals(1, bitVector.get(i), desc + i); } for (int i = start + count; i < length; i++) { - Assert.assertTrue(desc + i, bitVector.isNull(i)); + assertTrue(bitVector.isNull(i), desc + i); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVectorHelper.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVectorHelper.java index 1da4a4c4914b9..b1ef45c918b72 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVectorHelper.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVectorHelper.java @@ -17,16 +17,16 @@ package org.apache.arrow.vector; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.memory.util.MemoryUtil; import org.apache.arrow.vector.ipc.message.ArrowFieldNode; -import org.junit.Test; +import org.junit.jupiter.api.Test; public class TestBitVectorHelper { @Test diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java index 056b6bdd2b787..b38e046659669 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java @@ -17,9 +17,9 @@ package org.apache.arrow.vector; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.nio.charset.StandardCharsets; @@ -30,7 +30,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.CallBack; -import org.junit.Test; +import org.junit.jupiter.api.Test; public class TestBufferOwnershipTransfer { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java b/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java index 97de27bec8237..7d4d08636d740 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java @@ -18,9 +18,10 @@ package org.apache.arrow.vector; import static org.apache.arrow.vector.TestUtils.newVector; -import static org.junit.Assert.*; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.math.BigDecimal; import java.nio.charset.StandardCharsets; @@ -31,9 +32,9 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.types.Types.MinorType; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; /* * Tested field types: @@ -60,12 +61,12 @@ public class TestCopyFrom { private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new RootAllocator(Long.MAX_VALUE); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } @@ -99,10 +100,7 @@ public void testCopyFromWithNulls() { if (i % 3 == 0) { assertNull(vector.getObject(i)); } else { - assertEquals( - "unexpected value at index: " + i, - Integer.toString(i), - vector.getObject(i).toString()); + assertEquals(Integer.toString(i), vector.getObject(i).toString(), "unexpected value at index: " + i); } } @@ -116,10 +114,7 @@ public void testCopyFromWithNulls() { if (i % 3 == 0) { assertNull(vector2.getObject(i)); } else { - assertEquals( - "unexpected value at index: " + i, - Integer.toString(i), - vector2.getObject(i).toString()); + assertEquals(Integer.toString(i), vector2.getObject(i).toString(), "unexpected value at index: " + i); } } @@ -133,10 +128,7 @@ public void testCopyFromWithNulls() { if (i % 3 == 0) { assertNull(vector2.getObject(i)); } else { - assertEquals( - "unexpected value at index: " + i, - Integer.toString(i), - vector2.getObject(i).toString()); + assertEquals(Integer.toString(i), vector2.getObject(i).toString(), "unexpected value at index: " + i); } } } @@ -171,10 +163,7 @@ public void testCopyFromWithNulls1() { if (i % 3 == 0) { assertNull(vector.getObject(i)); } else { - assertEquals( - "unexpected value at index: " + i, - Integer.toString(i), - vector.getObject(i).toString()); + assertEquals(Integer.toString(i), vector.getObject(i).toString(), "unexpected value at index: " + i); } } @@ -192,10 +181,7 @@ public void testCopyFromWithNulls1() { if (i % 3 == 0) { assertNull(vector2.getObject(i)); } else { - assertEquals( - "unexpected value at index: " + i, - Integer.toString(i), - vector2.getObject(i).toString()); + assertEquals(Integer.toString(i), vector2.getObject(i).toString(), "unexpected value at index: " + i); } } @@ -209,10 +195,7 @@ public void testCopyFromWithNulls1() { if (i % 3 == 0) { assertNull(vector2.getObject(i)); } else { - assertEquals( - "unexpected value at index: " + i, - Integer.toString(i), - vector2.getObject(i).toString()); + assertEquals(Integer.toString(i), vector2.getObject(i).toString(), "unexpected value at index: " + i); } } } @@ -247,7 +230,7 @@ public void testCopyFromWithNulls2() { if ((i & 1) == 0) { assertNull(vector1.getObject(i)); } else { - assertEquals("unexpected value at index: " + i, 1000 + i, vector1.get(i)); + assertEquals(1000 + i, vector1.get(i), "unexpected value at index: " + i); } } @@ -274,7 +257,7 @@ public void testCopyFromWithNulls2() { if (((i & 1) == 0) || (i >= initialCapacity)) { assertNull(vector2.getObject(i)); } else { - assertEquals("unexpected value at index: " + i, 1000 + i, vector2.get(i)); + assertEquals(1000 + i, vector2.get(i), "unexpected value at index: " + i); } } } @@ -309,7 +292,7 @@ public void testCopyFromWithNulls3() { if ((i & 1) == 0) { assertNull(vector1.getObject(i)); } else { - assertEquals("unexpected value at index: " + i, 10000000000L + (long) i, vector1.get(i)); + assertEquals(10000000000L + (long) i, vector1.get(i), "unexpected value at index: " + i); } } @@ -336,7 +319,7 @@ public void testCopyFromWithNulls3() { if (((i & 1) == 0) || (i >= initialCapacity)) { assertNull(vector2.getObject(i)); } else { - assertEquals("unexpected value at index: " + i, 10000000000L + (long) i, vector2.get(i)); + assertEquals(10000000000L + (long) i, vector2.get(i), "unexpected value at index: " + i); } } } @@ -450,7 +433,7 @@ public void testCopyFromWithNulls5() { if ((i & 1) == 0) { assertNull(vector1.getObject(i)); } else { - assertEquals("unexpected value at index: " + i, 100.25f + (float) i, vector1.get(i), 0); + assertEquals(100.25f + (float) i, vector1.get(i), 0, "unexpected value at index: " + i); } } @@ -477,7 +460,7 @@ public void testCopyFromWithNulls5() { if (((i & 1) == 0) || (i >= initialCapacity)) { assertNull(vector2.getObject(i)); } else { - assertEquals("unexpected value at index: " + i, 100.25f + i * 1.0f, vector2.get(i), 0); + assertEquals(100.25f + i * 1.0f, vector2.get(i), 0, "unexpected value at index: " + i); } } } @@ -512,8 +495,7 @@ public void testCopyFromWithNulls6() { if ((i & 1) == 0) { assertNull(vector1.getObject(i)); } else { - assertEquals( - "unexpected value at index: " + i, 123456.7865 + (double) i, vector1.get(i), 0); + assertEquals(123456.7865 + (double) i, vector1.get(i), 0, "unexpected value at index: " + i); } } @@ -540,8 +522,7 @@ public void testCopyFromWithNulls6() { if (((i & 1) == 0) || (i >= initialCapacity)) { assertNull(vector2.getObject(i)); } else { - assertEquals( - "unexpected value at index: " + i, 123456.7865 + (double) i, vector2.get(i), 0); + assertEquals(123456.7865 + (double) i, vector2.get(i), 0, "unexpected value at index: " + i); } } } @@ -715,7 +696,7 @@ public void testCopyFromWithNulls9() { if ((i & 1) == 0) { assertNull(vector1.getObject(i)); } else { - assertEquals("unexpected value at index: " + i, val + (short) i, vector1.get(i)); + assertEquals(val + (short) i, vector1.get(i), "unexpected value at index: " + i); } } @@ -742,7 +723,7 @@ public void testCopyFromWithNulls9() { if (((i & 1) == 0) || (i >= initialCapacity)) { assertNull(vector2.getObject(i)); } else { - assertEquals("unexpected value at index: " + i, val + (short) i, vector2.get(i)); + assertEquals(val + (short) i, vector2.get(i), "unexpected value at index: " + i); } } } @@ -778,7 +759,7 @@ public void testCopyFromWithNulls10() { if ((i & 1) == 0) { assertNull(vector1.getObject(i)); } else { - assertEquals("unexpected value at index: " + i, val + (long) i, vector1.get(i)); + assertEquals(val + (long) i, vector1.get(i), "unexpected value at index: " + i); } } @@ -805,7 +786,7 @@ public void testCopyFromWithNulls10() { if (((i & 1) == 0) || (i >= initialCapacity)) { assertNull(vector2.getObject(i)); } else { - assertEquals("unexpected value at index: " + i, val + (long) i, vector2.get(i)); + assertEquals(val + (long) i, vector2.get(i), "unexpected value at index: " + i); } } } @@ -841,7 +822,7 @@ public void testCopyFromWithNulls11() { if ((i & 1) == 0) { assertNull(vector1.getObject(i)); } else { - assertEquals("unexpected value at index: " + i, val + i, vector1.get(i)); + assertEquals(val + i, vector1.get(i), "unexpected value at index: " + i); } } @@ -868,7 +849,7 @@ public void testCopyFromWithNulls11() { if (((i & 1) == 0) || (i >= initialCapacity)) { assertNull(vector2.getObject(i)); } else { - assertEquals("unexpected value at index: " + i, val + i, vector2.get(i)); + assertEquals(val + i, vector2.get(i), "unexpected value at index: " + i); } } } @@ -906,7 +887,7 @@ public void testCopyFromWithNulls12() { if ((i & 1) == 0) { assertNull(vector1.getObject(i)); } else { - assertEquals("unexpected value at index: " + i, val, vector1.get(i)); + assertEquals(val, vector1.get(i), "unexpected value at index: " + i); val++; } } @@ -934,7 +915,7 @@ public void testCopyFromWithNulls12() { if (((i & 1) == 0) || (i >= initialCapacity)) { assertNull(vector2.getObject(i)); } else { - assertEquals("unexpected value at index: " + i, val, vector2.get(i)); + assertEquals(val, vector2.get(i), "unexpected value at index: " + i); val++; } } @@ -1039,7 +1020,7 @@ public void testCopyFromWithNulls14() { if ((i & 1) == 0) { assertNull(vector1.getObject(i)); } else { - assertEquals("unexpected value at index: " + i, val + (long) i, vector1.get(i)); + assertEquals(val + (long) i, vector1.get(i), "unexpected value at index: " + i); } } @@ -1066,7 +1047,7 @@ public void testCopyFromWithNulls14() { if (((i & 1) == 0) || (i >= initialCapacity)) { assertNull(vector2.getObject(i)); } else { - assertEquals("unexpected value at index: " + i, val + (long) i, vector2.get(i)); + assertEquals(val + (long) i, vector2.get(i), "unexpected value at index: " + i); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimal256Vector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimal256Vector.java index fc5dfc38587a4..6886abcc63cdf 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimal256Vector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimal256Vector.java @@ -17,10 +17,10 @@ package org.apache.arrow.vector; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.math.BigDecimal; import java.math.BigInteger; @@ -29,9 +29,9 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.util.TransferPair; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestDecimal256Vector { @@ -49,12 +49,12 @@ public class TestDecimal256Vector { private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } @@ -80,7 +80,7 @@ public void testValuesWriteRead() { for (int i = 0; i < intValues.length; i++) { BigDecimal value = decimalVector.getObject(i); - assertEquals("unexpected data at index: " + i, values[i], value); + assertEquals(values[i], value, "unexpected data at index: " + i); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java index 572f13fea1ed1..c7a12fd6ac87c 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDecimalVector.java @@ -17,10 +17,10 @@ package org.apache.arrow.vector; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.math.BigDecimal; import java.math.BigInteger; @@ -29,9 +29,9 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.util.TransferPair; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestDecimalVector { @@ -49,12 +49,12 @@ public class TestDecimalVector { private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } @@ -80,7 +80,7 @@ public void testValuesWriteRead() { for (int i = 0; i < intValues.length; i++) { BigDecimal value = decimalVector.getObject(i); - assertEquals("unexpected data at index: " + i, values[i], value); + assertEquals(values[i], value, "unexpected data at index: " + i); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDenseUnionVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDenseUnionVector.java index 0621fd4527520..0b74f760d2941 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestDenseUnionVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDenseUnionVector.java @@ -17,10 +17,10 @@ package org.apache.arrow.vector; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.ArrayList; import java.util.HashMap; @@ -47,21 +47,21 @@ import org.apache.arrow.vector.util.JsonStringHashMap; import org.apache.arrow.vector.util.Text; import org.apache.arrow.vector.util.TransferPair; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestDenseUnionVector { private static final String EMPTY_SCHEMA_PATH = ""; private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } @@ -262,8 +262,8 @@ public void testSplitAndTransfer() throws Exception { /* check the toVector output after doing the splitAndTransfer */ for (int i = 0; i < length; i++) { - assertEquals("Different data at indexes: " + (start + i) + "and " + i, sourceVector.getObject(start + i), - toVector.getObject(i)); + assertEquals(sourceVector.getObject(start + i), toVector.getObject(i), + "Different data at indexes: " + (start + i) + "and " + i); } } } @@ -356,7 +356,8 @@ public void testSplitAndTransferWithMixedVectors() throws Exception { /* check the toVector output after doing the splitAndTransfer */ for (int i = 0; i < length; i++) { - assertEquals("Different values at index: " + i, sourceVector.getObject(start + i), toVector.getObject(i)); + assertEquals(sourceVector.getObject(start + i), toVector.getObject(i), + "Different values at index: " + i); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java index 9ffa79470eeb8..caccc2360e85c 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDictionaryVector.java @@ -19,7 +19,11 @@ import static org.apache.arrow.vector.TestUtils.*; import static org.apache.arrow.vector.testing.ValueVectorDataPopulator.setVector; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.nio.charset.StandardCharsets; import java.util.Arrays; @@ -49,9 +53,9 @@ import org.apache.arrow.vector.types.pojo.DictionaryEncoding; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.Text; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestDictionaryVector { @@ -63,12 +67,12 @@ public class TestDictionaryVector { byte[][] data = new byte[][] {zero, one, two}; - @Before + @BeforeEach public void init() { allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } @@ -896,7 +900,7 @@ public void testNoMemoryLeak() { assertEquals("Dictionary encoding not defined for value:" + new Text(two), e.getMessage()); } } - assertEquals("encode memory leak", 0, allocator.getAllocatedMemory()); + assertEquals(0, allocator.getAllocatedMemory(), "encode memory leak"); // test no memory leak when decode try (final IntVector indices = newVector(IntVector.class, "", Types.MinorType.INT, allocator); @@ -914,7 +918,7 @@ public void testNoMemoryLeak() { assertEquals("Provided dictionary does not contain value for index 3", e.getMessage()); } } - assertEquals("decode memory leak", 0, allocator.getAllocatedMemory()); + assertEquals(0, allocator.getAllocatedMemory(), "decode memory leak"); } @Test @@ -942,7 +946,7 @@ public void testListNoMemoryLeak() { assertEquals("Dictionary encoding not defined for value:20", e.getMessage()); } } - assertEquals("list encode memory leak", 0, allocator.getAllocatedMemory()); + assertEquals(0, allocator.getAllocatedMemory(), "list encode memory leak"); try (final ListVector indices = ListVector.empty("indices", allocator); final ListVector dictionaryVector = ListVector.empty("dict", allocator)) { @@ -966,7 +970,7 @@ public void testListNoMemoryLeak() { assertEquals("Provided dictionary does not contain value for index 3", e.getMessage()); } } - assertEquals("list decode memory leak", 0, allocator.getAllocatedMemory()); + assertEquals(0, allocator.getAllocatedMemory(), "list decode memory leak"); } @Test @@ -1003,7 +1007,7 @@ public void testStructNoMemoryLeak() { assertEquals("Dictionary encoding not defined for value:baz", e.getMessage()); } } - assertEquals("struct encode memory leak", 0, allocator.getAllocatedMemory()); + assertEquals(0, allocator.getAllocatedMemory(), "struct encode memory leak"); try (final StructVector indices = StructVector.empty("indices", allocator); final VarCharVector dictVector1 = new VarCharVector("f0", allocator); @@ -1040,7 +1044,7 @@ public void testStructNoMemoryLeak() { assertEquals("Provided dictionary does not contain value for index 3", e.getMessage()); } } - assertEquals("struct decode memory leak", 0, allocator.getAllocatedMemory()); + assertEquals(0, allocator.getAllocatedMemory(), "struct decode memory leak"); } private void testDictionary(Dictionary dictionary, ToIntBiFunction valGetter) { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestDurationVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestDurationVector.java index c5d4d296cc024..6ed44be849726 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestDurationVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestDurationVector.java @@ -17,9 +17,9 @@ package org.apache.arrow.vector; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import java.time.Duration; @@ -28,19 +28,19 @@ import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.util.TransferPair; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestDurationVector { RootAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100); } - @After + @AfterEach public void terminate() { allocator.close(); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeBinaryVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeBinaryVector.java index b9cd89e4ad731..4b52c7a41ff07 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeBinaryVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeBinaryVector.java @@ -17,8 +17,12 @@ package org.apache.arrow.vector; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; @@ -26,9 +30,9 @@ import org.apache.arrow.vector.holders.NullableFixedSizeBinaryHolder; import org.apache.arrow.vector.util.ReusableByteArray; import org.apache.arrow.vector.util.TransferPair; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestFixedSizeBinaryVector { private static final int numValues = 123; @@ -85,7 +89,7 @@ private static void failWithException(String message) throws Exception { } - @Before + @BeforeEach public void init() throws Exception { allocator = new DirtyRootAllocator(Integer.MAX_VALUE, (byte) 100); vector = new FixedSizeBinaryVector("fixedSizeBinary", allocator, typeWidth); @@ -128,7 +132,7 @@ public void init() throws Exception { largeNullableHolder.buffer = largeBuf; } - @After + @AfterEach public void terminate() throws Exception { for (int i = 0; i < numValues; i++) { bufs[i].close(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java index bde6dd491dd71..54ce8e2ae0e7c 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java @@ -17,11 +17,12 @@ package org.apache.arrow.vector; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.math.BigDecimal; import java.nio.ByteBuffer; @@ -41,21 +42,20 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.Text; import org.apache.arrow.vector.util.TransferPair; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestFixedSizeListVector { private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } @@ -76,12 +76,12 @@ public void testIntType() { UnionFixedSizeListReader reader = vector.getReader(); for (int i = 0; i < 10; i++) { reader.setPosition(i); - Assert.assertTrue(reader.isSet()); - Assert.assertTrue(reader.next()); + assertTrue(reader.isSet()); + assertTrue(reader.next()); assertEquals(i, reader.reader().readInteger().intValue()); - Assert.assertTrue(reader.next()); + assertTrue(reader.next()); assertEquals(i + 10, reader.reader().readInteger().intValue()); - Assert.assertFalse(reader.next()); + assertFalse(reader.next()); assertEquals(Arrays.asList(i, i + 10), reader.readObject()); } } @@ -107,16 +107,16 @@ public void testFloatTypeNullable() { for (int i = 0; i < 10; i++) { reader.setPosition(i); if (i % 2 == 0) { - Assert.assertTrue(reader.isSet()); - Assert.assertTrue(reader.next()); + assertTrue(reader.isSet()); + assertTrue(reader.next()); assertEquals(i + 0.1f, reader.reader().readFloat(), 0.00001); - Assert.assertTrue(reader.next()); + assertTrue(reader.next()); assertEquals(i + 10.1f, reader.reader().readFloat(), 0.00001); - Assert.assertFalse(reader.next()); + assertFalse(reader.next()); assertEquals(Arrays.asList(i + 0.1f, i + 10.1f), reader.readObject()); } else { - Assert.assertFalse(reader.isSet()); - Assert.assertNull(reader.readObject()); + assertFalse(reader.isSet()); + assertNull(reader.readObject()); } } } @@ -149,18 +149,18 @@ public void testNestedInList() { reader.setPosition(i); if (i % 2 == 0) { for (int j = 0; j < i % 7; j++) { - Assert.assertTrue(reader.next()); + assertTrue(reader.next()); FieldReader innerListReader = reader.reader(); for (int k = 0; k < 2; k++) { - Assert.assertTrue(innerListReader.next()); + assertTrue(innerListReader.next()); assertEquals(k + j, innerListReader.reader().readInteger().intValue()); } - Assert.assertFalse(innerListReader.next()); + assertFalse(innerListReader.next()); } - Assert.assertFalse(reader.next()); + assertFalse(reader.next()); } else { - Assert.assertFalse(reader.isSet()); - Assert.assertNull(reader.readObject()); + assertFalse(reader.isSet()); + assertNull(reader.readObject()); } } } @@ -196,40 +196,40 @@ public void testTransferPair() { UnionFixedSizeListReader reader = to.getReader(); reader.setPosition(0); - Assert.assertFalse(reader.isSet()); - Assert.assertNull(reader.readObject()); + assertFalse(reader.isSet()); + assertNull(reader.readObject()); reader.setPosition(1); - Assert.assertTrue(reader.isSet()); - Assert.assertTrue(reader.next()); + assertTrue(reader.isSet()); + assertTrue(reader.next()); assertEquals(0.1f, reader.reader().readFloat(), 0.00001); - Assert.assertTrue(reader.next()); + assertTrue(reader.next()); assertEquals(10.1f, reader.reader().readFloat(), 0.00001); - Assert.assertFalse(reader.next()); + assertFalse(reader.next()); assertEquals(Arrays.asList(0.1f, 10.1f), reader.readObject()); reader.setPosition(2); - Assert.assertTrue(reader.isSet()); - Assert.assertTrue(reader.next()); + assertTrue(reader.isSet()); + assertTrue(reader.next()); assertEquals(2.1f, reader.reader().readFloat(), 0.00001); - Assert.assertTrue(reader.next()); + assertTrue(reader.next()); assertEquals(12.1f, reader.reader().readFloat(), 0.00001); - Assert.assertFalse(reader.next()); + assertFalse(reader.next()); assertEquals(Arrays.asList(2.1f, 12.1f), reader.readObject()); reader.setPosition(3); - Assert.assertTrue(reader.isSet()); - Assert.assertTrue(reader.next()); + assertTrue(reader.isSet()); + assertTrue(reader.next()); assertEquals(4.1f, reader.reader().readFloat(), 0.00001); - Assert.assertTrue(reader.next()); + assertTrue(reader.next()); assertEquals(14.1f, reader.reader().readFloat(), 0.00001); - Assert.assertFalse(reader.next()); + assertFalse(reader.next()); assertEquals(Arrays.asList(4.1f, 14.1f), reader.readObject()); for (int i = 4; i < 10; i++) { reader.setPosition(i); - Assert.assertFalse(reader.isSet()); - Assert.assertNull(reader.readObject()); + assertFalse(reader.isSet()); + assertNull(reader.readObject()); } } } @@ -238,11 +238,11 @@ public void testTransferPair() { public void testConsistentChildName() throws Exception { try (FixedSizeListVector listVector = FixedSizeListVector.empty("sourceVector", /*size=*/2, allocator)) { String emptyListStr = listVector.getField().toString(); - Assert.assertTrue(emptyListStr.contains(ListVector.DATA_VECTOR_NAME)); + assertTrue(emptyListStr.contains(ListVector.DATA_VECTOR_NAME)); listVector.addOrGetVector(FieldType.nullable(MinorType.INT.getType())); String emptyVectorStr = listVector.getField().toString(); - Assert.assertTrue(emptyVectorStr.contains(ListVector.DATA_VECTOR_NAME)); + assertTrue(emptyVectorStr.contains(ListVector.DATA_VECTOR_NAME)); } } @@ -354,27 +354,29 @@ public void testDecimalIndexCheck() throws Exception { } - @Test(expected = IllegalStateException.class) + @Test public void testWriteIllegalData() throws Exception { - try (final FixedSizeListVector vector1 = FixedSizeListVector.empty("vector", /*size=*/3, allocator)) { + assertThrows(IllegalStateException.class, () -> { + try (final FixedSizeListVector vector1 = FixedSizeListVector.empty("vector", /*size=*/3, allocator)) { - UnionFixedSizeListWriter writer1 = vector1.getWriter(); - writer1.allocate(); + UnionFixedSizeListWriter writer1 = vector1.getWriter(); + writer1.allocate(); - int[] values1 = new int[] {1, 2, 3}; - int[] values2 = new int[] {4, 5, 6, 7, 8}; + int[] values1 = new int[]{1, 2, 3}; + int[] values2 = new int[]{4, 5, 6, 7, 8}; - //set some values - writeListVector(vector1, writer1, values1); - writeListVector(vector1, writer1, values2); - writer1.setValueCount(3); + //set some values + writeListVector(vector1, writer1, values1); + writeListVector(vector1, writer1, values2); + writer1.setValueCount(3); - assertEquals(3, vector1.getValueCount()); - int[] realValue1 = convertListToIntArray(vector1.getObject(0)); - assertTrue(Arrays.equals(values1, realValue1)); - int[] realValue2 = convertListToIntArray(vector1.getObject(1)); - assertTrue(Arrays.equals(values2, realValue2)); - } + assertEquals(3, vector1.getValueCount()); + int[] realValue1 = convertListToIntArray(vector1.getObject(0)); + assertTrue(Arrays.equals(values1, realValue1)); + int[] realValue2 = convertListToIntArray(vector1.getObject(1)); + assertTrue(Arrays.equals(values2, realValue2)); + } + }); } @Test diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestIntervalMonthDayNanoVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestIntervalMonthDayNanoVector.java index 681897b93c12c..82bf1dd423b5e 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestIntervalMonthDayNanoVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestIntervalMonthDayNanoVector.java @@ -17,8 +17,7 @@ package org.apache.arrow.vector; -import static org.junit.Assert.assertEquals; - +import static org.junit.jupiter.api.Assertions.assertEquals; import java.time.Duration; import java.time.Period; @@ -29,20 +28,20 @@ import org.apache.arrow.vector.types.IntervalUnit; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.ArrowType; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestIntervalMonthDayNanoVector { private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestIntervalYearVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestIntervalYearVector.java index 4b2ae2eb3d49b..6cb72f38307df 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestIntervalYearVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestIntervalYearVector.java @@ -17,28 +17,28 @@ package org.apache.arrow.vector; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertSame; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.types.IntervalUnit; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.util.TransferPair; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestIntervalYearVector { private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java index ffd87c99d508d..d4bb3d4c97bcf 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeListVector.java @@ -17,11 +17,11 @@ package org.apache.arrow.vector; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.ArrayList; import java.util.Arrays; @@ -39,21 +39,20 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.TransferPair; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestLargeListVector { private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } @@ -91,11 +90,11 @@ public void testCopyFrom() throws Exception { // assert the output vector is correct FieldReader reader = outVector.getReader(); - Assert.assertTrue("shouldn't be null", reader.isSet()); + assertTrue(reader.isSet(), "shouldn't be null"); reader.setPosition(1); - Assert.assertFalse("should be null", reader.isSet()); + assertFalse(reader.isSet(), "should be null"); reader.setPosition(2); - Assert.assertTrue("shouldn't be null", reader.isSet()); + assertTrue(reader.isSet(), "shouldn't be null"); /* index 0 */ @@ -433,15 +432,15 @@ public void testSplitAndTransfer() throws Exception { dataLength2 = (int) toOffsetBuffer.getLong((i + 1) * LargeListVector.OFFSET_WIDTH) - (int) toOffsetBuffer.getLong(i * LargeListVector.OFFSET_WIDTH); - assertEquals("Different data lengths at index: " + i + " and start: " + start, - dataLength1, dataLength2); + assertEquals(dataLength1, dataLength2, + "Different data lengths at index: " + i + " and start: " + start); offset1 = (int) offsetBuffer.getLong((start + i) * LargeListVector.OFFSET_WIDTH); offset2 = (int) toOffsetBuffer.getLong(i * LargeListVector.OFFSET_WIDTH); for (int j = 0; j < dataLength1; j++) { - assertEquals("Different data at indexes: " + offset1 + " and " + offset2, - dataVector.getObject(offset1), dataVector1.getObject(offset2)); + assertEquals(dataVector.getObject(offset1), dataVector1.getObject(offset2), + "Different data at indexes: " + offset1 + " and " + offset2); offset1++; offset2++; diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVarBinaryVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVarBinaryVector.java index 36607903b01a2..3a51cca51706c 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVarBinaryVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVarBinaryVector.java @@ -17,10 +17,10 @@ package org.apache.arrow.vector; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.nio.charset.StandardCharsets; import java.util.Arrays; @@ -32,20 +32,20 @@ import org.apache.arrow.vector.holders.NullableLargeVarBinaryHolder; import org.apache.arrow.vector.util.ReusableByteArray; import org.apache.arrow.vector.util.TransferPair; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestLargeVarBinaryVector { private BufferAllocator allocator; - @Before + @BeforeEach public void prepare() { allocator = new RootAllocator(Integer.MAX_VALUE); } - @After + @AfterEach public void shutdown() { allocator.close(); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVarCharVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVarCharVector.java index 06b27a9eba156..aa9c7fed38a6b 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVarCharVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestLargeVarCharVector.java @@ -17,12 +17,14 @@ package org.apache.arrow.vector; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -41,11 +43,9 @@ import org.apache.arrow.vector.util.OversizedAllocationException; import org.apache.arrow.vector.util.Text; import org.apache.arrow.vector.util.TransferPair; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestLargeVarCharVector { @@ -58,12 +58,12 @@ public class TestLargeVarCharVector { private BufferAllocator allocator; - @Before + @BeforeEach public void prepare() { allocator = new RootAllocator(Integer.MAX_VALUE); } - @After + @AfterEach public void shutdown() { allocator.close(); } @@ -162,7 +162,7 @@ public void testInvalidStartIndex() { final TransferPair tp = largeVarCharVector.makeTransferPair(newLargeVarCharVector); - IllegalArgumentException e = Assertions.assertThrows( + IllegalArgumentException e = assertThrows( IllegalArgumentException.class, () -> tp.splitAndTransfer(valueCount, 10)); @@ -181,7 +181,7 @@ public void testInvalidLength() { final TransferPair tp = largeVarCharVector.makeTransferPair(newLargeVarCharVector); - IllegalArgumentException e = Assertions.assertThrows( + IllegalArgumentException e = assertThrows( IllegalArgumentException.class, () -> tp.splitAndTransfer(0, valueCount * 2)); @@ -298,39 +298,43 @@ public void testSetLastSetUsage() { } } - @Test(expected = OutOfMemoryException.class) + @Test public void testVectorAllocateNew() { - try (RootAllocator smallAllocator = new RootAllocator(200); - LargeVarCharVector vector = new LargeVarCharVector("vec", smallAllocator)) { - vector.allocateNew(); - } + assertThrows(OutOfMemoryException.class, () -> { + try (RootAllocator smallAllocator = new RootAllocator(200); + LargeVarCharVector vector = new LargeVarCharVector("vec", smallAllocator)) { + vector.allocateNew(); + } + }); } - @Test(expected = OversizedAllocationException.class) + @Test public void testLargeVariableVectorReallocation() { - final LargeVarCharVector vector = new LargeVarCharVector("vector", allocator); - // edge case 1: value count = MAX_VALUE_ALLOCATION - final long expectedAllocationInBytes = BaseValueVector.MAX_ALLOCATION_SIZE; - final int expectedOffsetSize = 10; - try { - vector.allocateNew(expectedAllocationInBytes, 10); - assertTrue(expectedOffsetSize <= vector.getValueCapacity()); - assertTrue(expectedAllocationInBytes <= vector.getDataBuffer().capacity()); - vector.reAlloc(); - assertTrue(expectedOffsetSize * 2 <= vector.getValueCapacity()); - assertTrue(expectedAllocationInBytes * 2 <= vector.getDataBuffer().capacity()); - } finally { - vector.close(); - } + assertThrows(OversizedAllocationException.class, () -> { + final LargeVarCharVector vector = new LargeVarCharVector("vector", allocator); + // edge case 1: value count = MAX_VALUE_ALLOCATION + final long expectedAllocationInBytes = BaseValueVector.MAX_ALLOCATION_SIZE; + final int expectedOffsetSize = 10; + try { + vector.allocateNew(expectedAllocationInBytes, 10); + assertTrue(expectedOffsetSize <= vector.getValueCapacity()); + assertTrue(expectedAllocationInBytes <= vector.getDataBuffer().capacity()); + vector.reAlloc(); + assertTrue(expectedOffsetSize * 2 <= vector.getValueCapacity()); + assertTrue(expectedAllocationInBytes * 2 <= vector.getDataBuffer().capacity()); + } finally { + vector.close(); + } - // common: value count < MAX_VALUE_ALLOCATION - try { - vector.allocateNew(BaseValueVector.MAX_ALLOCATION_SIZE / 2, 0); - vector.reAlloc(); // value allocation reaches to MAX_VALUE_ALLOCATION - vector.reAlloc(); // this tests if it overflows - } finally { - vector.close(); - } + // common: value count < MAX_VALUE_ALLOCATION + try { + vector.allocateNew(BaseValueVector.MAX_ALLOCATION_SIZE / 2, 0); + vector.reAlloc(); // value allocation reaches to MAX_VALUE_ALLOCATION + vector.reAlloc(); // this tests if it overflows + } finally { + vector.close(); + } + }); } @Test @@ -784,7 +788,7 @@ public void testNullableType() { try { vector.set(initialCapacity, "foo".getBytes(StandardCharsets.UTF_8)); - Assert.fail("Expected out of bounds exception"); + fail("Expected out of bounds exception"); } catch (Exception e) { // ok } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java index 97f2d9fd6def1..cbcb6cf9d7963 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java @@ -17,12 +17,12 @@ package org.apache.arrow.vector; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.ArrayList; import java.util.Arrays; @@ -44,21 +44,20 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.TransferPair; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestListVector { private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } @@ -96,11 +95,11 @@ public void testCopyFrom() throws Exception { // assert the output vector is correct FieldReader reader = outVector.getReader(); - Assert.assertTrue("shouldn't be null", reader.isSet()); + assertTrue(reader.isSet(), "shouldn't be null"); reader.setPosition(1); - Assert.assertFalse("should be null", reader.isSet()); + assertFalse(reader.isSet(), "should be null"); reader.setPosition(2); - Assert.assertTrue("shouldn't be null", reader.isSet()); + assertTrue(reader.isSet(), "shouldn't be null"); /* index 0 */ @@ -439,15 +438,15 @@ public void testSplitAndTransfer() throws Exception { dataLength2 = toOffsetBuffer.getInt((i + 1) * ListVector.OFFSET_WIDTH) - toOffsetBuffer.getInt(i * ListVector.OFFSET_WIDTH); - assertEquals("Different data lengths at index: " + i + " and start: " + start, - dataLength1, dataLength2); + assertEquals(dataLength1, dataLength2, + "Different data lengths at index: " + i + " and start: " + start); offset1 = offsetBuffer.getInt((start + i) * ListVector.OFFSET_WIDTH); offset2 = toOffsetBuffer.getInt(i * ListVector.OFFSET_WIDTH); for (int j = 0; j < dataLength1; j++) { - assertEquals("Different data at indexes: " + offset1 + " and " + offset2, - dataVector.getObject(offset1), dataVector1.getObject(offset2)); + assertEquals(dataVector.getObject(offset1), dataVector1.getObject(offset2), + "Different data at indexes: " + offset1 + " and " + offset2); offset1++; offset2++; diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestNullCheckingForGet.java b/java/vector/src/test/java/org/apache/arrow/vector/TestNullCheckingForGet.java index f1345e88ab8b9..51ad470bb6417 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestNullCheckingForGet.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestNullCheckingForGet.java @@ -17,11 +17,13 @@ package org.apache.arrow.vector; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.lang.reflect.Field; import java.net.URLClassLoader; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; /** * Test cases for {@link NullCheckingForGet}. @@ -63,7 +65,7 @@ public void testDefaultValue() throws Exception { ClassLoader classLoader = copyClassLoader(); if (classLoader != null) { boolean nullCheckingEnabled = getFlagValue(classLoader); - Assert.assertTrue(nullCheckingEnabled); + assertTrue(nullCheckingEnabled); } } @@ -79,7 +81,7 @@ public void testEnableSysProperty() throws Exception { ClassLoader classLoader = copyClassLoader(); if (classLoader != null) { boolean nullCheckingEnabled = getFlagValue(classLoader); - Assert.assertFalse(nullCheckingEnabled); + assertFalse(nullCheckingEnabled); } // restore system property diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestOutOfMemoryForValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestOutOfMemoryForValueVector.java index 7f26b5c1b79f6..200786f54a92d 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestOutOfMemoryForValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestOutOfMemoryForValueVector.java @@ -17,12 +17,14 @@ package org.apache.arrow.vector; +import static org.junit.jupiter.api.Assertions.assertThrows; + import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.OutOfMemoryException; import org.apache.arrow.memory.RootAllocator; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; /** * This class tests cases where we expect to receive {@link OutOfMemoryException}. @@ -33,40 +35,48 @@ public class TestOutOfMemoryForValueVector { private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new RootAllocator(200); // Start with low memory limit } - @Test(expected = OutOfMemoryException.class) + @Test public void variableWidthVectorAllocateNew() { - try (VarCharVector vector = new VarCharVector(EMPTY_SCHEMA_PATH, allocator)) { - vector.allocateNew(); - } + assertThrows(OutOfMemoryException.class, () -> { + try (VarCharVector vector = new VarCharVector(EMPTY_SCHEMA_PATH, allocator)) { + vector.allocateNew(); + } + }); } - @Test(expected = OutOfMemoryException.class) + @Test public void variableWidthVectorAllocateNewCustom() { - try (VarCharVector vector = new VarCharVector(EMPTY_SCHEMA_PATH, allocator)) { - vector.allocateNew(2342, 234); - } + assertThrows(OutOfMemoryException.class, () -> { + try (VarCharVector vector = new VarCharVector(EMPTY_SCHEMA_PATH, allocator)) { + vector.allocateNew(2342, 234); + } + }); } - @Test(expected = OutOfMemoryException.class) + @Test public void fixedWidthVectorAllocateNew() { - try (IntVector vector = new IntVector(EMPTY_SCHEMA_PATH, allocator)) { - vector.allocateNew(); - } + assertThrows(OutOfMemoryException.class, () -> { + try (IntVector vector = new IntVector(EMPTY_SCHEMA_PATH, allocator)) { + vector.allocateNew(); + } + }); } - @Test(expected = OutOfMemoryException.class) + @Test public void fixedWidthVectorAllocateNewCustom() { - try (IntVector vector = new IntVector(EMPTY_SCHEMA_PATH, allocator)) { - vector.allocateNew(2342); - } + assertThrows(OutOfMemoryException.class, () -> { + try (IntVector vector = new IntVector(EMPTY_SCHEMA_PATH, allocator)) { + vector.allocateNew(2342); + } + }); } - @After + @AfterEach public void terminate() { allocator.close(); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java index 23414e9f5df1c..f89828e4ceeb2 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestOversizedAllocationForValueVector.java @@ -18,15 +18,16 @@ package org.apache.arrow.vector; import static org.apache.arrow.memory.util.LargeMemoryUtil.checkedCastToInt; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.util.OversizedAllocationException; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; /** * This class tests that OversizedAllocationException occurs when a large memory is allocated for a vector. @@ -39,94 +40,100 @@ public class TestOversizedAllocationForValueVector { private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new RootAllocator(Long.MAX_VALUE); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } - @Test(expected = OversizedAllocationException.class) + @Test public void testFixedVectorReallocation() { - final UInt4Vector vector = new UInt4Vector(EMPTY_SCHEMA_PATH, allocator); - // edge case 1: buffer size = max value capacity - final int expectedValueCapacity = checkedCastToInt(BaseValueVector.MAX_ALLOCATION_SIZE / 4); - try { - vector.allocateNew(expectedValueCapacity); - assertEquals(expectedValueCapacity, vector.getValueCapacity()); - vector.reAlloc(); - assertEquals(expectedValueCapacity * 2, vector.getValueCapacity()); - } finally { - vector.close(); - } + assertThrows(OversizedAllocationException.class, () -> { + final UInt4Vector vector = new UInt4Vector(EMPTY_SCHEMA_PATH, allocator); + // edge case 1: buffer size = max value capacity + final int expectedValueCapacity = checkedCastToInt(BaseValueVector.MAX_ALLOCATION_SIZE / 4); + try { + vector.allocateNew(expectedValueCapacity); + assertEquals(expectedValueCapacity, vector.getValueCapacity()); + vector.reAlloc(); + assertEquals(expectedValueCapacity * 2, vector.getValueCapacity()); + } finally { + vector.close(); + } - // common case: value count < max value capacity - try { - vector.allocateNew(checkedCastToInt(BaseValueVector.MAX_ALLOCATION_SIZE / 8)); - vector.reAlloc(); // value allocation reaches to MAX_VALUE_ALLOCATION - vector.reAlloc(); // this should throw an IOOB - } finally { - vector.close(); - } + // common case: value count < max value capacity + try { + vector.allocateNew(checkedCastToInt(BaseValueVector.MAX_ALLOCATION_SIZE / 8)); + vector.reAlloc(); // value allocation reaches to MAX_VALUE_ALLOCATION + vector.reAlloc(); // this should throw an IOOB + } finally { + vector.close(); + } + }); } - @Test(expected = OversizedAllocationException.class) + @Test public void testBitVectorReallocation() { - final BitVector vector = new BitVector(EMPTY_SCHEMA_PATH, allocator); - // edge case 1: buffer size ~ max value capacity - final int expectedValueCapacity = 1 << 29; - try { - vector.allocateNew(expectedValueCapacity); - assertEquals(expectedValueCapacity, vector.getValueCapacity()); - vector.reAlloc(); - assertEquals(expectedValueCapacity * 2, vector.getValueCapacity()); - } finally { - vector.close(); - } + assertThrows(OversizedAllocationException.class, () -> { + final BitVector vector = new BitVector(EMPTY_SCHEMA_PATH, allocator); + // edge case 1: buffer size ~ max value capacity + final int expectedValueCapacity = 1 << 29; + try { + vector.allocateNew(expectedValueCapacity); + assertEquals(expectedValueCapacity, vector.getValueCapacity()); + vector.reAlloc(); + assertEquals(expectedValueCapacity * 2, vector.getValueCapacity()); + } finally { + vector.close(); + } - // common: value count < MAX_VALUE_ALLOCATION - try { - vector.allocateNew(expectedValueCapacity); - for (int i = 0; i < 3; i++) { - vector.reAlloc(); // expand buffer size + // common: value count < MAX_VALUE_ALLOCATION + try { + vector.allocateNew(expectedValueCapacity); + for (int i = 0; i < 3; i++) { + vector.reAlloc(); // expand buffer size + } + assertEquals(Integer.MAX_VALUE, vector.getValueCapacity()); + vector.reAlloc(); // buffer size ~ max allocation + assertEquals(Integer.MAX_VALUE, vector.getValueCapacity()); + vector.reAlloc(); // overflow + } finally { + vector.close(); } - assertEquals(Integer.MAX_VALUE, vector.getValueCapacity()); - vector.reAlloc(); // buffer size ~ max allocation - assertEquals(Integer.MAX_VALUE, vector.getValueCapacity()); - vector.reAlloc(); // overflow - } finally { - vector.close(); - } + }); } - @Test(expected = OversizedAllocationException.class) + @Test public void testVariableVectorReallocation() { - final VarCharVector vector = new VarCharVector(EMPTY_SCHEMA_PATH, allocator); - // edge case 1: value count = MAX_VALUE_ALLOCATION - final long expectedAllocationInBytes = BaseValueVector.MAX_ALLOCATION_SIZE; - final int expectedOffsetSize = 10; - try { - vector.allocateNew(expectedAllocationInBytes, 10); - assertTrue(expectedOffsetSize <= vector.getValueCapacity()); - assertTrue(expectedAllocationInBytes <= vector.getDataBuffer().capacity()); - vector.reAlloc(); - assertTrue(expectedOffsetSize * 2 <= vector.getValueCapacity()); - assertTrue(expectedAllocationInBytes * 2 <= vector.getDataBuffer().capacity()); - } finally { - vector.close(); - } + assertThrows(OversizedAllocationException.class, () -> { + final VarCharVector vector = new VarCharVector(EMPTY_SCHEMA_PATH, allocator); + // edge case 1: value count = MAX_VALUE_ALLOCATION + final long expectedAllocationInBytes = BaseValueVector.MAX_ALLOCATION_SIZE; + final int expectedOffsetSize = 10; + try { + vector.allocateNew(expectedAllocationInBytes, 10); + assertTrue(expectedOffsetSize <= vector.getValueCapacity()); + assertTrue(expectedAllocationInBytes <= vector.getDataBuffer().capacity()); + vector.reAlloc(); + assertTrue(expectedOffsetSize * 2 <= vector.getValueCapacity()); + assertTrue(expectedAllocationInBytes * 2 <= vector.getDataBuffer().capacity()); + } finally { + vector.close(); + } - // common: value count < MAX_VALUE_ALLOCATION - try { - vector.allocateNew(BaseValueVector.MAX_ALLOCATION_SIZE / 2, 0); - vector.reAlloc(); // value allocation reaches to MAX_VALUE_ALLOCATION - vector.reAlloc(); // this tests if it overflows - } finally { - vector.close(); - } + // common: value count < MAX_VALUE_ALLOCATION + try { + vector.allocateNew(BaseValueVector.MAX_ALLOCATION_SIZE / 2, 0); + vector.reAlloc(); // value allocation reaches to MAX_VALUE_ALLOCATION + vector.reAlloc(); // this tests if it overflows + } finally { + vector.close(); + } + }); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestPeriodDuration.java b/java/vector/src/test/java/org/apache/arrow/vector/TestPeriodDuration.java index 2b9f4cca8c22f..bf4cda6b4271a 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestPeriodDuration.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestPeriodDuration.java @@ -17,8 +17,8 @@ package org.apache.arrow.vector; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import java.time.Duration; import java.time.LocalDate; @@ -26,7 +26,7 @@ import java.time.Period; import java.time.temporal.ChronoUnit; -import org.junit.Test; +import org.junit.jupiter.api.Test; public class TestPeriodDuration { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java index 68f5e14dabb9b..ccb2890863314 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestStructVector.java @@ -17,7 +17,11 @@ package org.apache.arrow.vector; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import java.util.ArrayList; import java.util.HashMap; @@ -39,21 +43,20 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.TransferPair; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestStructVector { private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } @@ -64,7 +67,7 @@ public void testFieldMetadata() throws Exception { metadata.put("k1", "v1"); FieldType type = new FieldType(true, Struct.INSTANCE, null, metadata); try (StructVector vector = new StructVector("struct", allocator, type, null)) { - Assert.assertEquals(vector.getField().getMetadata(), type.getMetadata()); + assertEquals(vector.getField().getMetadata(), type.getMetadata()); } } @@ -108,8 +111,8 @@ public void testAllocateAfterReAlloc() throws Exception { /* * Verify that the buffer sizes haven't changed. */ - Assert.assertEquals(vector.getValidityBuffer().capacity(), savedValidityBufferCapacity); - Assert.assertEquals(vector.getValueCapacity(), savedValueCapacity); + assertEquals(vector.getValidityBuffer().capacity(), savedValidityBufferCapacity); + assertEquals(vector.getValueCapacity(), savedValueCapacity); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java index 1b0387feb73ff..10298112ddc98 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java @@ -17,11 +17,11 @@ package org.apache.arrow.vector; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.ArrayList; import java.util.HashMap; @@ -44,21 +44,21 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.TransferPair; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestUnionVector { private static final String EMPTY_SCHEMA_PATH = ""; private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } @@ -283,8 +283,8 @@ public void testSplitAndTransfer() throws Exception { /* check the toVector output after doing the splitAndTransfer */ for (int i = 0; i < length; i++) { - assertEquals("Different data at indexes: " + (start + i) + "and " + i, sourceVector.getObject(start + i), - toVector.getObject(i)); + assertEquals(sourceVector.getObject(start + i), toVector.getObject(i), + "Different data at indexes: " + (start + i) + "and " + i); } } } @@ -373,7 +373,8 @@ public void testSplitAndTransferWithMixedVectors() throws Exception { /* check the toVector output after doing the splitAndTransfer */ for (int i = 0; i < length; i++) { - assertEquals("Different values at index: " + i, sourceVector.getObject(start + i), toVector.getObject(i)); + assertEquals(sourceVector.getObject(start + i), toVector.getObject(i), + "Different values at index: " + i); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharListVector.java index bfe489fa5af4e..6d4e64837adbc 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharListVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVarCharListVector.java @@ -17,6 +17,8 @@ package org.apache.arrow.vector; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.nio.charset.StandardCharsets; import org.apache.arrow.memory.ArrowBuf; @@ -25,21 +27,20 @@ import org.apache.arrow.vector.complex.impl.UnionListWriter; import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.FieldType; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestVarCharListVector { private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } @@ -72,8 +73,8 @@ public void testVarCharListWithNulls() { writer.setValueCount(2); - Assert.assertEquals(2, vector.getValueCount()); - Assert.assertEquals(2, vector.getDataVector().getValueCount()); + assertEquals(2, vector.getValueCount()); + assertEquals(2, vector.getDataVector().getValueCount()); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorAlloc.java index b96f6ab6afedd..02a85faa20cd6 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorAlloc.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorAlloc.java @@ -17,8 +17,8 @@ package org.apache.arrow.vector; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Arrays; import java.util.Collections; @@ -39,23 +39,23 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestVectorAlloc { private BufferAllocator rootAllocator; private BufferAllocator policyAllocator; - @Before + @BeforeEach public void init() { rootAllocator = new RootAllocator(Long.MAX_VALUE); policyAllocator = new RootAllocator(AllocationListener.NOOP, Integer.MAX_VALUE, new CustomPolicy()); } - @After + @AfterEach public void terminate() throws Exception { rootAllocator.close(); policyAllocator.close(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java index 9043bd4f8f2d4..21cbefae45161 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java @@ -17,7 +17,10 @@ package org.apache.arrow.vector; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.nio.charset.StandardCharsets; @@ -37,22 +40,21 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.DataSizeRoundingUtil; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestVectorReAlloc { private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new RootAllocator(Long.MAX_VALUE); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } @@ -68,7 +70,7 @@ public void testFixedType() { try { vector.set(initialCapacity, 0); - Assert.fail("Expected out of bounds exception"); + fail("Expected out of bounds exception"); } catch (Exception e) { // ok } @@ -92,7 +94,7 @@ public void testNullableType() { try { vector.set(initialCapacity, "foo".getBytes(StandardCharsets.UTF_8)); - Assert.fail("Expected out of bounds exception"); + fail("Expected out of bounds exception"); } catch (Exception e) { // ok } @@ -101,7 +103,7 @@ public void testNullableType() { assertTrue(vector.getValueCapacity() >= 2 * initialCapacity); vector.set(initialCapacity, "foo".getBytes(StandardCharsets.UTF_8)); - assertEquals("foo", new String(vector.get(initialCapacity), StandardCharsets.UTF_8)); + assertEquals(new String(vector.get(initialCapacity), StandardCharsets.UTF_8), "foo"); } } @@ -117,7 +119,7 @@ public void testListType() { try { vector.getInnerValueCountAt(2014); - Assert.fail("Expected out of bounds exception"); + fail("Expected out of bounds exception"); } catch (Exception e) { // ok } @@ -140,7 +142,7 @@ public void testStructType() { try { vector.getObject(513); - Assert.fail("Expected out of bounds exception"); + fail("Expected out of bounds exception"); } catch (Exception e) { // ok } @@ -161,7 +163,7 @@ public void testVariableWidthTypeSetNullValues() { for (int i = 0; i < numNullValues1; i++) { v1.setNull(i); } - Assert.assertTrue(v1.getBufferSizeFor(numNullValues1) > 0); + assertTrue(v1.getBufferSizeFor(numNullValues1) > 0); } try (final BaseLargeVariableWidthVector v2 = new LargeVarCharVector("var2", allocator)) { @@ -171,7 +173,7 @@ public void testVariableWidthTypeSetNullValues() { for (int i = 0; i < numNullValues2; i++) { v2.setNull(i); } - Assert.assertTrue(v2.getBufferSizeFor(numNullValues2) > 0); + assertTrue(v2.getBufferSizeFor(numNullValues2) > 0); } } @@ -194,7 +196,7 @@ public void testFixedAllocateAfterReAlloc() throws Exception { /* * Verify that the buffer sizes haven't changed. */ - Assert.assertEquals(vector.getValueCapacity(), savedValueCapacity); + assertEquals(vector.getValueCapacity(), savedValueCapacity); } } @@ -218,8 +220,8 @@ public void testVariableAllocateAfterReAlloc() throws Exception { /* * Verify that the buffer sizes haven't changed. */ - Assert.assertEquals(vector.getValueCapacity(), savedValueCapacity); - Assert.assertEquals(vector.valueBuffer.capacity(), savedValueBufferSize); + assertEquals(vector.getValueCapacity(), savedValueCapacity); + assertEquals(vector.valueBuffer.capacity(), savedValueBufferSize); } } @@ -243,8 +245,8 @@ public void testLargeVariableAllocateAfterReAlloc() throws Exception { /* * Verify that the buffer sizes haven't changed. */ - Assert.assertEquals(vector.getValueCapacity(), savedValueCapacity); - Assert.assertEquals(vector.valueBuffer.capacity(), savedValueBufferSize); + assertEquals(vector.getValueCapacity(), savedValueCapacity); + assertEquals(vector.valueBuffer.capacity(), savedValueBufferSize); } } @@ -256,8 +258,8 @@ public void testVarCharAllocateNew() throws Exception { vector.allocateNew(count); // verify that the validity buffer and value buffer have capacity for at least 'count' elements. - Assert.assertTrue(vector.getValidityBuffer().capacity() >= DataSizeRoundingUtil.divideBy8Ceil(count)); - Assert.assertTrue(vector.getOffsetBuffer().capacity() >= (count + 1) * BaseVariableWidthVector.OFFSET_WIDTH); + assertTrue(vector.getValidityBuffer().capacity() >= DataSizeRoundingUtil.divideBy8Ceil(count)); + assertTrue(vector.getOffsetBuffer().capacity() >= (count + 1) * BaseVariableWidthVector.OFFSET_WIDTH); } } @@ -269,8 +271,8 @@ public void testLargeVarCharAllocateNew() throws Exception { vector.allocateNew(count); // verify that the validity buffer and value buffer have capacity for at least 'count' elements. - Assert.assertTrue(vector.getValidityBuffer().capacity() >= DataSizeRoundingUtil.divideBy8Ceil(count)); - Assert.assertTrue(vector.getOffsetBuffer().capacity() >= (count + 1) * BaseLargeVariableWidthVector.OFFSET_WIDTH); + assertTrue(vector.getValidityBuffer().capacity() >= DataSizeRoundingUtil.divideBy8Ceil(count)); + assertTrue(vector.getOffsetBuffer().capacity() >= (count + 1) * BaseLargeVariableWidthVector.OFFSET_WIDTH); } } @@ -282,8 +284,8 @@ public void testVarCharAllocateNewUsingHelper() throws Exception { AllocationHelper.allocateNew(vector, count); // verify that the validity buffer and value buffer have capacity for at least 'count' elements. - Assert.assertTrue(vector.getValidityBuffer().capacity() >= DataSizeRoundingUtil.divideBy8Ceil(count)); - Assert.assertTrue(vector.getOffsetBuffer().capacity() >= (count + 1) * BaseVariableWidthVector.OFFSET_WIDTH); + assertTrue(vector.getValidityBuffer().capacity() >= DataSizeRoundingUtil.divideBy8Ceil(count)); + assertTrue(vector.getOffsetBuffer().capacity() >= (count + 1) * BaseVariableWidthVector.OFFSET_WIDTH); } } @@ -295,8 +297,8 @@ public void testLargeVarCharAllocateNewUsingHelper() throws Exception { AllocationHelper.allocateNew(vector, count); // verify that the validity buffer and value buffer have capacity for at least 'count' elements. - Assert.assertTrue(vector.getValidityBuffer().capacity() >= DataSizeRoundingUtil.divideBy8Ceil(count)); - Assert.assertTrue(vector.getOffsetBuffer().capacity() >= (count + 1) * BaseLargeVariableWidthVector.OFFSET_WIDTH); + assertTrue(vector.getValidityBuffer().capacity() >= DataSizeRoundingUtil.divideBy8Ceil(count)); + assertTrue(vector.getOffsetBuffer().capacity() >= (count + 1) * BaseLargeVariableWidthVector.OFFSET_WIDTH); } } @@ -314,7 +316,7 @@ public void testFixedRepeatedClearAndSet() throws Exception { } // should be deterministic, and not cause a run-away increase in capacity. - Assert.assertEquals(vector.getValueCapacity(), savedValueCapacity); + assertEquals(vector.getValueCapacity(), savedValueCapacity); } } @@ -333,7 +335,7 @@ public void testVariableRepeatedClearAndSet() throws Exception { } // should be deterministic, and not cause a run-away increase in capacity. - Assert.assertEquals(vector.getValueCapacity(), savedValueCapacity); + assertEquals(vector.getValueCapacity(), savedValueCapacity); } } @@ -359,7 +361,7 @@ public void testRepeatedValueVectorClearAndSet() throws Exception { } // should be deterministic, and not cause a run-away increase in capacity. - Assert.assertEquals(vector.getValueCapacity(), savedValueCapacity); + assertEquals(vector.getValueCapacity(), savedValueCapacity); } } @@ -386,7 +388,7 @@ public void testStructVectorClearAndSet() throws Exception { } // should be deterministic, and not cause a run-away increase in capacity. - Assert.assertEquals(vector.getValueCapacity(), savedValueCapacity); + assertEquals(vector.getValueCapacity(), savedValueCapacity); } } @@ -415,7 +417,7 @@ public void testFixedSizeListVectorClearAndSet() { } // should be deterministic, and not cause a run-away increase in capacity. - Assert.assertEquals(vector.getValueCapacity(), savedValueCapacity); + assertEquals(vector.getValueCapacity(), savedValueCapacity); } } @@ -440,7 +442,7 @@ public void testUnionVectorClearAndSet() { } // should be deterministic, and not cause a run-away increase in capacity. - Assert.assertEquals(vector.getValueCapacity(), savedValueCapacity); + assertEquals(vector.getValueCapacity(), savedValueCapacity); } } @@ -468,7 +470,7 @@ public void testDenseUnionVectorClearAndSet() { } // should be deterministic, and not cause a run-away increase in capacity. - Assert.assertEquals(vector.getValueCapacity(), savedValueCapacity); + assertEquals(vector.getValueCapacity(), savedValueCapacity); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java index 19700e02161c7..2a6f86426ae8a 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReset.java @@ -17,8 +17,8 @@ package org.apache.arrow.vector; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.nio.charset.StandardCharsets; @@ -34,20 +34,20 @@ import org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeList; import org.apache.arrow.vector.types.pojo.ArrowType.Int; import org.apache.arrow.vector.types.pojo.FieldType; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestVectorReset { private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new RootAllocator(Long.MAX_VALUE); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorSchemaRoot.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorSchemaRoot.java index 207962eb45b85..76500052fa632 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorSchemaRoot.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorSchemaRoot.java @@ -17,10 +17,11 @@ package org.apache.arrow.vector; -import static junit.framework.TestCase.assertTrue; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.ArrayList; import java.util.Arrays; @@ -35,20 +36,20 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestVectorSchemaRoot { private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new RootAllocator(Long.MAX_VALUE); } - @After + @AfterEach public void terminate() { allocator.close(); } @@ -226,20 +227,22 @@ public void testSlice() { } } - @Test(expected = IllegalArgumentException.class) + @Test public void testSliceWithInvalidParam() { - try (final IntVector intVector = new IntVector("intVector", allocator); - final Float4Vector float4Vector = new Float4Vector("float4Vector", allocator)) { - intVector.setValueCount(10); - float4Vector.setValueCount(10); - for (int i = 0; i < 10; i++) { - intVector.setSafe(i, i); - float4Vector.setSafe(i, i + 0.1f); - } - final VectorSchemaRoot original = new VectorSchemaRoot(Arrays.asList(intVector, float4Vector)); + assertThrows(IllegalArgumentException.class, () -> { + try (final IntVector intVector = new IntVector("intVector", allocator); + final Float4Vector float4Vector = new Float4Vector("float4Vector", allocator)) { + intVector.setValueCount(10); + float4Vector.setValueCount(10); + for (int i = 0; i < 10; i++) { + intVector.setSafe(i, i); + float4Vector.setSafe(i, i + 0.1f); + } + final VectorSchemaRoot original = new VectorSchemaRoot(Arrays.asList(intVector, float4Vector)); - original.slice(0, 20); - } + original.slice(0, 20); + } + }); } @Test diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java index eac72f4b2c893..82ae5c038cbc2 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java @@ -18,9 +18,9 @@ package org.apache.arrow.vector; import static java.util.Arrays.asList; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.ArrayList; @@ -44,21 +44,20 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestVectorUnloadLoad { private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new RootAllocator(Long.MAX_VALUE); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } @@ -116,9 +115,9 @@ public void testUnloadLoad() throws IOException { FieldReader bigIntReader = newRoot.getVector("bigInt").getReader(); for (int i = 0; i < count; i++) { intReader.setPosition(i); - Assert.assertEquals(i, intReader.readInteger().intValue()); + assertEquals(i, intReader.readInteger().intValue()); bigIntReader.setPosition(i); - Assert.assertEquals(i, bigIntReader.readLong().longValue()); + assertEquals(i, bigIntReader.readLong().longValue()); } } } @@ -188,7 +187,7 @@ public void testUnloadLoadAddPadding() throws IOException { for (int j = 0; j < i % 4 + 1; j++) { expected.add(i); } - Assert.assertEquals(expected, reader.readObject()); + assertEquals(expected, reader.readObject()); } } @@ -256,9 +255,9 @@ public void testLoadValidityBuffer() throws IOException { IntVector intDefinedVector = (IntVector) newRoot.getVector("intDefined"); IntVector intNullVector = (IntVector) newRoot.getVector("intNull"); for (int i = 0; i < count; i++) { - assertFalse("#" + i, intDefinedVector.isNull(i)); - assertEquals("#" + i, i, intDefinedVector.get(i)); - assertTrue("#" + i, intNullVector.isNull(i)); + assertFalse(intDefinedVector.isNull(i), "#" + i); + assertEquals(i, intDefinedVector.get(i), "#" + i); + assertTrue(intNullVector.isNull(i), "#" + i); } intDefinedVector.setSafe(count + 10, 1234); assertTrue(intDefinedVector.isNull(count + 1)); @@ -319,13 +318,13 @@ public void testUnloadLoadDuplicates() throws IOException { vectorLoader.load(recordBatch); List targets = newRoot.getFieldVectors(); - Assert.assertEquals(sources.size(), targets.size()); + assertEquals(sources.size(), targets.size()); for (int k = 0; k < sources.size(); k++) { IntVector src = (IntVector) sources.get(k); IntVector tgt = (IntVector) targets.get(k); - Assert.assertEquals(src.getValueCount(), tgt.getValueCount()); + assertEquals(src.getValueCount(), tgt.getValueCount()); for (int i = 0; i < count; i++) { - Assert.assertEquals(src.get(i), tgt.get(i)); + assertEquals(src.get(i), tgt.get(i)); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/compare/TestTypeEqualsVisitor.java b/java/vector/src/test/java/org/apache/arrow/vector/compare/TestTypeEqualsVisitor.java index 736b0f1b1aeac..6ff81faba73e8 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/compare/TestTypeEqualsVisitor.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/compare/TestTypeEqualsVisitor.java @@ -17,8 +17,8 @@ package org.apache.arrow.vector.compare; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.nio.charset.StandardCharsets; import java.util.HashMap; @@ -41,20 +41,20 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestTypeEqualsVisitor { private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new RootAllocator(Long.MAX_VALUE); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/TestDenseUnionBufferSize.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/TestDenseUnionBufferSize.java index 82ef7a479d05c..0e24fd0af6806 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/TestDenseUnionBufferSize.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/TestDenseUnionBufferSize.java @@ -17,7 +17,8 @@ package org.apache.arrow.vector.complex; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java index 29f25170332a2..67bdb9945fc94 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java @@ -17,8 +17,8 @@ package org.apache.arrow.vector.complex.impl; -import static org.junit.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.math.BigDecimal; @@ -39,9 +39,9 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.DecimalUtility; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestComplexCopier { @@ -49,12 +49,12 @@ public class TestComplexCopier { private static final int COUNT = 100; - @Before + @BeforeEach public void init() { allocator = new RootAllocator(Long.MAX_VALUE); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } @@ -90,7 +90,6 @@ public void testCopyFixedSizeListVector() { // validate equals assertTrue(VectorEqualsVisitor.vectorEquals(from, to)); - } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java index b7fc681c16118..3a54d539c290a 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java @@ -17,9 +17,9 @@ package org.apache.arrow.vector.complex.impl; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import java.nio.ByteBuffer; @@ -50,21 +50,21 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.Text; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestPromotableWriter { private static final String EMPTY_SCHEMA_PATH = ""; private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new DirtyRootAllocator(Long.MAX_VALUE, (byte) 100); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } @@ -128,33 +128,33 @@ public void testPromoteToUnion() throws Exception { final UnionVector uv = v.getChild("A", UnionVector.class); - assertFalse("0 shouldn't be null", uv.isNull(0)); + assertFalse(uv.isNull(0), "0 shouldn't be null"); assertEquals(false, uv.getObject(0)); - assertFalse("1 shouldn't be null", uv.isNull(1)); + assertFalse(uv.isNull(1), "1 shouldn't be null"); assertEquals(true, uv.getObject(1)); - assertFalse("2 shouldn't be null", uv.isNull(2)); + assertFalse(uv.isNull(2), "2 shouldn't be null"); assertEquals(10, uv.getObject(2)); - assertNull("3 should be null", uv.getObject(3)); + assertNull(uv.getObject(3), "3 should be null"); - assertFalse("4 shouldn't be null", uv.isNull(4)); + assertFalse(uv.isNull(4), "4 shouldn't be null"); assertEquals(100, uv.getObject(4)); - assertFalse("5 shouldn't be null", uv.isNull(5)); + assertFalse(uv.isNull(5), "5 shouldn't be null"); assertEquals(123123L, uv.getObject(5)); - assertFalse("6 shouldn't be null", uv.isNull(6)); + assertFalse(uv.isNull(6), "6 shouldn't be null"); NullableTimeStampMilliTZHolder readBackHolder = new NullableTimeStampMilliTZHolder(); uv.getTimeStampMilliTZVector().get(6, readBackHolder); assertEquals(12345L, readBackHolder.value); assertEquals("UTC", readBackHolder.timezone); - assertFalse("7 shouldn't be null", uv.isNull(7)); + assertFalse(uv.isNull(7), "7 shouldn't be null"); assertEquals(444413L, ((java.time.Duration) uv.getObject(7)).getSeconds()); - assertFalse("8 shouldn't be null", uv.isNull(8)); + assertFalse(uv.isNull(8), "8 shouldn't be null"); assertEquals(18978, ByteBuffer.wrap(uv.getFixedSizeBinaryVector().get(8)).order(ByteOrder.nativeOrder()).getInt()); @@ -172,10 +172,10 @@ public void testPromoteToUnion() throws Exception { Field childField1 = container.getField().getChildren().get(0).getChildren().get(0); Field childField2 = container.getField().getChildren().get(0).getChildren().get(1); - assertEquals("Child field should be union type: " + - childField1.getName(), ArrowTypeID.Union, childField1.getType().getTypeID()); - assertEquals("Child field should be decimal type: " + - childField2.getName(), ArrowTypeID.Decimal, childField2.getType().getTypeID()); + assertEquals(ArrowTypeID.Union, childField1.getType().getTypeID(), + "Child field should be union type: " + childField1.getName()); + assertEquals(ArrowTypeID.Decimal, childField2.getType().getTypeID(), + "Child field should be decimal type: " + childField2.getName()); buf.close(); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java index 19f0ea9d4e392..c7ed893d4c340 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java @@ -17,7 +17,12 @@ package org.apache.arrow.vector.complex.writer; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.math.BigDecimal; import java.nio.ByteBuffer; @@ -92,10 +97,9 @@ import org.apache.arrow.vector.util.JsonStringHashMap; import org.apache.arrow.vector.util.Text; import org.apache.arrow.vector.util.TransferPair; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestComplexWriter { @@ -103,12 +107,12 @@ public class TestComplexWriter { private static final int COUNT = 100; - @Before + @BeforeEach public void init() { allocator = new RootAllocator(Integer.MAX_VALUE); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } @@ -119,8 +123,8 @@ public void simpleNestedTypes() { StructReader rootReader = new SingleStructReaderImpl(parent).reader("root"); for (int i = 0; i < COUNT; i++) { rootReader.setPosition(i); - Assert.assertEquals(i, rootReader.reader("int").readInteger().intValue()); - Assert.assertEquals(i, rootReader.reader("bigInt").readLong().longValue()); + assertEquals(i, rootReader.reader("int").readInteger().intValue()); + assertEquals(i, rootReader.reader("bigInt").readLong().longValue()); } parent.close(); @@ -210,15 +214,15 @@ private void checkNullableStruct(NonNullableStructVector structVector) { StructReader rootReader = new SingleStructReaderImpl(structVector).reader("root"); for (int i = 0; i < COUNT; i++) { rootReader.setPosition(i); - assertTrue("index is set: " + i, rootReader.isSet()); + assertTrue(rootReader.isSet(), "index is set: " + i); FieldReader struct = rootReader.reader("struct"); if (i % 2 == 0) { - assertTrue("index is set: " + i, struct.isSet()); - assertNotNull("index is set: " + i, struct.readObject()); + assertTrue(struct.isSet(), "index is set: " + i); + assertNotNull(struct.readObject(), "index is set: " + i); assertEquals(i, struct.reader("nested").readLong().longValue()); } else { - assertFalse("index is not set: " + i, struct.isSet()); - assertNull("index is not set: " + i, struct.readObject()); + assertFalse(struct.isSet(), "index is not set: " + i); + assertNull(struct.readObject(), "index is not set: " + i); } } } @@ -245,11 +249,11 @@ public void testList() { StructReader rootReader = new SingleStructReaderImpl(parent).reader("root"); rootReader.setPosition(0); - assertTrue("row 0 list is not set", rootReader.reader("list").isSet()); + assertTrue(rootReader.reader("list").isSet(), "row 0 list is not set"); assertEquals(Long.valueOf(0), rootReader.reader("list").reader().readLong()); rootReader.setPosition(1); - assertFalse("row 1 list is set", rootReader.reader("list").isSet()); + assertFalse(rootReader.reader("list").isSet(), "row 1 list is set"); } } @@ -312,9 +316,9 @@ public void testListScalarNull() { for (int j = 0; j < i % 7; j++) { listReader.next(); if (j % 2 == 0) { - assertFalse("index is set: " + j, listReader.reader().isSet()); + assertFalse(listReader.reader().isSet(), "index is set: " + j); } else { - assertTrue("index is not set: " + j, listReader.reader().isSet()); + assertTrue(listReader.reader().isSet(), "index is not set: " + j); assertEquals(j, listReader.reader().readInteger().intValue()); } } @@ -392,7 +396,7 @@ public void listTimeStampMilliTZType() { for (int j = 0; j < i % 7; j++) { listReader.next(); if (j % 2 == 0) { - assertFalse("index is set: " + j, listReader.reader().isSet()); + assertFalse(listReader.reader().isSet(), "index is set: " + j); } else { NullableTimeStampMilliTZHolder actual = new NullableTimeStampMilliTZHolder(); listReader.reader().read(actual); @@ -430,7 +434,7 @@ public void listDurationType() { for (int j = 0; j < i % 7; j++) { listReader.next(); if (j % 2 == 0) { - assertFalse("index is set: " + j, listReader.reader().isSet()); + assertFalse(listReader.reader().isSet(), "index is set: " + j); } else { NullableDurationHolder actual = new NullableDurationHolder(); listReader.reader().read(actual); @@ -472,7 +476,7 @@ public void listFixedSizeBinaryType() throws Exception { for (int j = 0; j < i % 7; j++) { listReader.next(); if (j % 2 == 0) { - assertFalse("index is set: " + j, listReader.reader().isSet()); + assertFalse(listReader.reader().isSet(), "index is set: " + j); } else { NullableFixedSizeBinaryHolder actual = new NullableFixedSizeBinaryHolder(); listReader.reader().read(actual); @@ -505,11 +509,11 @@ public void listScalarTypeNullable() { for (int i = 0; i < COUNT; i++) { listReader.setPosition(i); if (i % 2 == 0) { - assertTrue("index is set: " + i, listReader.isSet()); - assertEquals("correct length at: " + i, i % 7, ((List) listReader.readObject()).size()); + assertTrue(listReader.isSet(), "index is set: " + i); + assertEquals(i % 7, ((List) listReader.readObject()).size(), "correct length at: " + i); } else { - assertFalse("index is not set: " + i, listReader.isSet()); - assertNull("index is not set: " + i, listReader.readObject()); + assertFalse(listReader.isSet(), "index is not set: " + i); + assertNull(listReader.readObject(), "index is not set: " + i); } } } @@ -537,8 +541,8 @@ public void listStructType() { listReader.setPosition(i); for (int j = 0; j < i % 7; j++) { listReader.next(); - Assert.assertEquals("record: " + i, j, listReader.reader().reader("int").readInteger().intValue()); - Assert.assertEquals(j, listReader.reader().reader("bigInt").readLong().longValue()); + assertEquals(j, listReader.reader().reader("int").readInteger().intValue(), "record: " + i); + assertEquals(j, listReader.reader().reader("bigInt").readLong().longValue()); } } } @@ -601,7 +605,7 @@ private void checkListOfLists(final ListVector listVector) { FieldReader innerListReader = listReader.reader(); for (int k = 0; k < i % 13; k++) { innerListReader.next(); - Assert.assertEquals("record: " + i, k, innerListReader.reader().readInteger().intValue()); + assertEquals(k, innerListReader.reader().readInteger().intValue(), "record: " + i); } } } @@ -673,9 +677,9 @@ private void checkUnionList(ListVector listVector) { for (int k = 0; k < i % 13; k++) { innerListReader.next(); if (k % 2 == 0) { - Assert.assertEquals("record: " + i, k, innerListReader.reader().readInteger().intValue()); + assertEquals(k, innerListReader.reader().readInteger().intValue(), "record: " + i); } else { - Assert.assertEquals("record: " + i, k, innerListReader.reader().readLong().longValue()); + assertEquals(k, innerListReader.reader().readLong().longValue(), "record: " + i); } } } @@ -724,11 +728,11 @@ private void checkListMap(ListVector listVector) { UnionMapReader mapReader = (UnionMapReader) listReader.reader(); for (int k = 0; k < i % 13; k++) { mapReader.next(); - Assert.assertEquals("record key: " + i, k, mapReader.key().readInteger().intValue()); + assertEquals(k, mapReader.key().readInteger().intValue(), "record key: " + i); if (k % 2 == 0) { - Assert.assertEquals("record value: " + i, k, mapReader.value().readLong().longValue()); + assertEquals(k, mapReader.value().readLong().longValue(), "record value: " + i); } else { - Assert.assertNull("record value: " + i, mapReader.value().readLong()); + assertNull(mapReader.value().readLong(), "record value: " + i); } } } @@ -772,24 +776,24 @@ public void simpleUnion() throws Exception { for (int i = 0; i < COUNT; i++) { unionReader.setPosition(i); if (i % 5 == 0) { - Assert.assertEquals(i, unionReader.readInteger().intValue()); + assertEquals(i, unionReader.readInteger().intValue()); } else if (i % 5 == 1) { NullableTimeStampMilliTZHolder holder = new NullableTimeStampMilliTZHolder(); unionReader.read(holder); - Assert.assertEquals(i, holder.value); - Assert.assertEquals("AsdfTimeZone", holder.timezone); + assertEquals(i, holder.value); + assertEquals("AsdfTimeZone", holder.timezone); } else if (i % 5 == 2) { NullableDurationHolder holder = new NullableDurationHolder(); unionReader.read(holder); - Assert.assertEquals(i, holder.value); - Assert.assertEquals(TimeUnit.NANOSECOND, holder.unit); + assertEquals(i, holder.value); + assertEquals(TimeUnit.NANOSECOND, holder.unit); } else if (i % 5 == 3) { NullableFixedSizeBinaryHolder holder = new NullableFixedSizeBinaryHolder(); unionReader.read(holder); assertEquals(i, holder.buffer.getInt(0)); assertEquals(4, holder.byteWidth); } else { - Assert.assertEquals((float) i, unionReader.readFloat(), 1e-12); + assertEquals((float) i, unionReader.readFloat(), 1e-12); } } vector.close(); @@ -808,12 +812,12 @@ public void promotableWriter() { bigIntWriter.writeBigInt(i); } Field field = parent.getField().getChildren().get(0).getChildren().get(0); - Assert.assertEquals("a", field.getName()); - Assert.assertEquals(Int.TYPE_TYPE, field.getType().getTypeID()); + assertEquals("a", field.getName()); + assertEquals(Int.TYPE_TYPE, field.getType().getTypeID()); Int intType = (Int) field.getType(); - Assert.assertEquals(64, intType.getBitWidth()); - Assert.assertTrue(intType.getIsSigned()); + assertEquals(64, intType.getBitWidth()); + assertTrue(intType.getIsSigned()); for (int i = 100; i < 200; i++) { VarCharWriter varCharWriter = rootWriter.varChar("a"); varCharWriter.setPosition(i); @@ -824,23 +828,23 @@ public void promotableWriter() { tempBuf.close(); } field = parent.getField().getChildren().get(0).getChildren().get(0); - Assert.assertEquals("a", field.getName()); - Assert.assertEquals(Union.TYPE_TYPE, field.getType().getTypeID()); - Assert.assertEquals(Int.TYPE_TYPE, field.getChildren().get(0).getType().getTypeID()); - Assert.assertEquals(Utf8.TYPE_TYPE, field.getChildren().get(1).getType().getTypeID()); + assertEquals("a", field.getName()); + assertEquals(Union.TYPE_TYPE, field.getType().getTypeID()); + assertEquals(Int.TYPE_TYPE, field.getChildren().get(0).getType().getTypeID()); + assertEquals(Utf8.TYPE_TYPE, field.getChildren().get(1).getType().getTypeID()); StructReader rootReader = new SingleStructReaderImpl(parent).reader("root"); for (int i = 0; i < 100; i++) { rootReader.setPosition(i); FieldReader reader = rootReader.reader("a"); Long value = reader.readLong(); - Assert.assertNotNull("index: " + i, value); - Assert.assertEquals(i, value.intValue()); + assertNotNull(value, "index: " + i); + assertEquals(i, value.intValue()); } for (int i = 100; i < 200; i++) { rootReader.setPosition(i); FieldReader reader = rootReader.reader("a"); Text value = reader.readText(); - Assert.assertEquals(Integer.toString(i), value.toString()); + assertEquals(Integer.toString(i), value.toString()); } } } @@ -857,14 +861,14 @@ public void promotableWriterSchema() { rootWriter.varChar("a"); Field field = parent.getField().getChildren().get(0).getChildren().get(0); - Assert.assertEquals("a", field.getName()); - Assert.assertEquals(ArrowTypeID.Union, field.getType().getTypeID()); + assertEquals("a", field.getName()); + assertEquals(ArrowTypeID.Union, field.getType().getTypeID()); - Assert.assertEquals(ArrowTypeID.Int, field.getChildren().get(0).getType().getTypeID()); + assertEquals(ArrowTypeID.Int, field.getChildren().get(0).getType().getTypeID()); Int intType = (Int) field.getChildren().get(0).getType(); - Assert.assertEquals(64, intType.getBitWidth()); - Assert.assertTrue(intType.getIsSigned()); - Assert.assertEquals(ArrowTypeID.Utf8, field.getChildren().get(1).getType().getTypeID()); + assertEquals(64, intType.getBitWidth()); + assertTrue(intType.getIsSigned()); + assertEquals(ArrowTypeID.Utf8, field.getChildren().get(1).getType().getTypeID()); } } @@ -901,18 +905,18 @@ public void structWriterMixedCaseFieldNames() { List fieldsCaseSensitive = parent.getField().getChildren().get(0).getChildren(); Set fieldNamesCaseSensitive = getFieldNames(fieldsCaseSensitive); - Assert.assertEquals(11, fieldNamesCaseSensitive.size()); - Assert.assertTrue(fieldNamesCaseSensitive.contains("int_field")); - Assert.assertTrue(fieldNamesCaseSensitive.contains("Int_Field")); - Assert.assertTrue(fieldNamesCaseSensitive.contains("float_field")); - Assert.assertTrue(fieldNamesCaseSensitive.contains("Float_Field")); - Assert.assertTrue(fieldNamesCaseSensitive.contains("struct_field")); - Assert.assertTrue(fieldNamesCaseSensitive.contains("struct_field::char_field")); - Assert.assertTrue(fieldNamesCaseSensitive.contains("struct_field::Char_Field")); - Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field")); - Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$")); - Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$::bit_field")); - Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$::Bit_Field")); + assertEquals(11, fieldNamesCaseSensitive.size()); + assertTrue(fieldNamesCaseSensitive.contains("int_field")); + assertTrue(fieldNamesCaseSensitive.contains("Int_Field")); + assertTrue(fieldNamesCaseSensitive.contains("float_field")); + assertTrue(fieldNamesCaseSensitive.contains("Float_Field")); + assertTrue(fieldNamesCaseSensitive.contains("struct_field")); + assertTrue(fieldNamesCaseSensitive.contains("struct_field::char_field")); + assertTrue(fieldNamesCaseSensitive.contains("struct_field::Char_Field")); + assertTrue(fieldNamesCaseSensitive.contains("list_field")); + assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$")); + assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$::bit_field")); + assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$::Bit_Field")); // test case-insensitive StructWriter ComplexWriter writerCaseInsensitive = new ComplexWriterImpl("rootCaseInsensitive", parent, false, false); @@ -932,14 +936,14 @@ public void structWriterMixedCaseFieldNames() { List fieldsCaseInsensitive = parent.getField().getChildren().get(1).getChildren(); Set fieldNamesCaseInsensitive = getFieldNames(fieldsCaseInsensitive); - Assert.assertEquals(7, fieldNamesCaseInsensitive.size()); - Assert.assertTrue(fieldNamesCaseInsensitive.contains("int_field")); - Assert.assertTrue(fieldNamesCaseInsensitive.contains("float_field")); - Assert.assertTrue(fieldNamesCaseInsensitive.contains("struct_field")); - Assert.assertTrue(fieldNamesCaseInsensitive.contains("struct_field::char_field")); - Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field")); - Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$")); - Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$::bit_field")); + assertEquals(7, fieldNamesCaseInsensitive.size()); + assertTrue(fieldNamesCaseInsensitive.contains("int_field")); + assertTrue(fieldNamesCaseInsensitive.contains("float_field")); + assertTrue(fieldNamesCaseInsensitive.contains("struct_field")); + assertTrue(fieldNamesCaseInsensitive.contains("struct_field::char_field")); + assertTrue(fieldNamesCaseSensitive.contains("list_field")); + assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$")); + assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$::bit_field")); } } @@ -976,15 +980,15 @@ public void timeStampSecWriter() throws Exception { FieldReader secReader = rootReader.reader("sec"); secReader.setPosition(0); LocalDateTime secDateTime = secReader.readLocalDateTime(); - Assert.assertEquals(expectedSecDateTime, secDateTime); + assertEquals(expectedSecDateTime, secDateTime); long secLong = secReader.readLong(); - Assert.assertEquals(expectedSecs, secLong); + assertEquals(expectedSecs, secLong); } { FieldReader secTZReader = rootReader.reader("secTZ"); secTZReader.setPosition(1); long secTZLong = secTZReader.readLong(); - Assert.assertEquals(expectedSecs, secTZLong); + assertEquals(expectedSecs, secTZLong); } } } @@ -1022,27 +1026,27 @@ public void timeStampMilliWriters() throws Exception { FieldReader milliReader = rootReader.reader("milli"); milliReader.setPosition(0); LocalDateTime milliDateTime = milliReader.readLocalDateTime(); - Assert.assertEquals(expectedMilliDateTime, milliDateTime); + assertEquals(expectedMilliDateTime, milliDateTime); long milliLong = milliReader.readLong(); - Assert.assertEquals(expectedMillis, milliLong); + assertEquals(expectedMillis, milliLong); } { FieldReader milliTZReader = rootReader.reader("milliTZ"); milliTZReader.setPosition(0); long milliTZLong = milliTZReader.readLong(); - Assert.assertEquals(expectedMillis, milliTZLong); + assertEquals(expectedMillis, milliTZLong); } } } private void checkTimestampField(Field field, String name) { - Assert.assertEquals(name, field.getName()); - Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, field.getType().getTypeID()); + assertEquals(name, field.getName()); + assertEquals(ArrowType.Timestamp.TYPE_TYPE, field.getType().getTypeID()); } private void checkTimestampTZField(Field field, String name, String tz) { checkTimestampField(field, name); - Assert.assertEquals(tz, ((Timestamp) field.getType()).getTimezone()); + assertEquals(tz, ((Timestamp) field.getType()).getTimezone()); } @Test @@ -1079,15 +1083,15 @@ public void timeStampMicroWriters() throws Exception { FieldReader microReader = rootReader.reader("micro"); microReader.setPosition(0); LocalDateTime microDateTime = microReader.readLocalDateTime(); - Assert.assertEquals(expectedMicroDateTime, microDateTime); + assertEquals(expectedMicroDateTime, microDateTime); long microLong = microReader.readLong(); - Assert.assertEquals(expectedMicros, microLong); + assertEquals(expectedMicros, microLong); } { FieldReader microReader = rootReader.reader("microTZ"); microReader.setPosition(1); long microLong = microReader.readLong(); - Assert.assertEquals(expectedMicros, microLong); + assertEquals(expectedMicros, microLong); } } } @@ -1125,18 +1129,18 @@ public void timeStampNanoWriters() throws Exception { FieldReader nanoReader = rootReader.reader("nano"); nanoReader.setPosition(0); LocalDateTime nanoDateTime = nanoReader.readLocalDateTime(); - Assert.assertEquals(expectedNanoDateTime, nanoDateTime); + assertEquals(expectedNanoDateTime, nanoDateTime); long nanoLong = nanoReader.readLong(); - Assert.assertEquals(expectedNanos, nanoLong); + assertEquals(expectedNanos, nanoLong); } { FieldReader nanoReader = rootReader.reader("nanoTZ"); nanoReader.setPosition(0); long nanoLong = nanoReader.readLong(); - Assert.assertEquals(expectedNanos, nanoLong); + assertEquals(expectedNanos, nanoLong); NullableTimeStampNanoTZHolder h = new NullableTimeStampNanoTZHolder(); nanoReader.read(h); - Assert.assertEquals(expectedNanos, h.value); + assertEquals(expectedNanos, h.value); } } @@ -1173,8 +1177,8 @@ public void fixedSizeBinaryWriters() throws Exception { // schema List children = parent.getField().getChildren().get(0).getChildren(); - Assert.assertEquals(fieldName, children.get(0).getName()); - Assert.assertEquals(ArrowType.FixedSizeBinary.TYPE_TYPE, children.get(0).getType().getTypeID()); + assertEquals(fieldName, children.get(0).getName()); + assertEquals(ArrowType.FixedSizeBinary.TYPE_TYPE, children.get(0).getType().getTypeID()); // read StructReader rootReader = new SingleStructReaderImpl(parent).reader("root"); @@ -1183,7 +1187,7 @@ public void fixedSizeBinaryWriters() throws Exception { for (int i = 0; i < numValues; i++) { fixedSizeBinaryReader.setPosition(i); byte[] readValues = fixedSizeBinaryReader.readByteArray(); - Assert.assertArrayEquals(values[i], readValues); + assertArrayEquals(values[i], readValues); } } @@ -1369,17 +1373,17 @@ public void testListWriterWithNulls() { for (int i = 0; i < COUNT; i++) { listReader.setPosition(i); if (i % 2 == 0) { - Assert.assertTrue(listReader.isSet()); + assertTrue(listReader.isSet()); listReader.next(); if (i % 4 == 0) { - Assert.assertNull(listReader.reader().readInteger()); + assertNull(listReader.reader().readInteger()); } else { - Assert.assertEquals(i, listReader.reader().readInteger().intValue()); + assertEquals(i, listReader.reader().readInteger().intValue()); listReader.next(); - Assert.assertEquals(i * 2, listReader.reader().readInteger().intValue()); + assertEquals(i * 2, listReader.reader().readInteger().intValue()); } } else { - Assert.assertFalse(listReader.isSet()); + assertFalse(listReader.isSet()); } } } @@ -1419,20 +1423,20 @@ public void testListOfListWriterWithNulls() { for (int i = 0; i < COUNT; i++) { listReader.setPosition(i); if (i % 2 == 0) { - Assert.assertTrue(listReader.isSet()); + assertTrue(listReader.isSet()); listReader.next(); if (i % 4 == 0) { - Assert.assertFalse(listReader.reader().isSet()); + assertFalse(listReader.reader().isSet()); } else { listReader.reader().next(); - Assert.assertFalse(listReader.reader().reader().isSet()); + assertFalse(listReader.reader().reader().isSet()); listReader.reader().next(); - Assert.assertEquals(i, listReader.reader().reader().readInteger().intValue()); + assertEquals(i, listReader.reader().reader().readInteger().intValue()); listReader.reader().next(); - Assert.assertEquals(i * 2, listReader.reader().reader().readInteger().intValue()); + assertEquals(i * 2, listReader.reader().reader().readInteger().intValue()); } } else { - Assert.assertFalse(listReader.isSet()); + assertFalse(listReader.isSet()); } } } @@ -1478,23 +1482,23 @@ public void testListOfListOfListWriterWithNulls() { for (int i = 0; i < COUNT; i++) { listReader.setPosition(i); if (i % 4 == 0) { - Assert.assertFalse(listReader.isSet()); + assertFalse(listReader.isSet()); } else { - Assert.assertTrue(listReader.isSet()); + assertTrue(listReader.isSet()); listReader.next(); if (i % 4 == 1) { - Assert.assertFalse(listReader.reader().isSet()); + assertFalse(listReader.reader().isSet()); } else if (i % 4 == 2) { listReader.reader().next(); - Assert.assertFalse(listReader.reader().reader().isSet()); + assertFalse(listReader.reader().reader().isSet()); } else { listReader.reader().next(); listReader.reader().reader().next(); - Assert.assertFalse(listReader.reader().reader().reader().isSet()); + assertFalse(listReader.reader().reader().reader().isSet()); listReader.reader().reader().next(); - Assert.assertEquals(i, listReader.reader().reader().reader().readInteger().intValue()); + assertEquals(i, listReader.reader().reader().reader().readInteger().intValue()); listReader.reader().reader().next(); - Assert.assertEquals(i * 2, listReader.reader().reader().reader().readInteger().intValue()); + assertEquals(i * 2, listReader.reader().reader().reader().readInteger().intValue()); } } } @@ -1507,7 +1511,7 @@ public void testStructOfList() { structVector.addOrGetList("childList1"); NullableStructReaderImpl structReader = structVector.getReader(); FieldReader childListReader = structReader.reader("childList1"); - Assert.assertNotNull(childListReader); + assertNotNull(childListReader); } try (StructVector structVector = StructVector.empty("struct2", allocator)) { @@ -1523,9 +1527,9 @@ public void testStructOfList() { NullableStructReaderImpl structReader = structVector.getReader(); FieldReader childListReader = structReader.reader("childList2"); int size = childListReader.size(); - Assert.assertEquals(1, size); + assertEquals(1, size); int data = childListReader.reader().readInteger(); - Assert.assertEquals(10, data); + assertEquals(10, data); } try (StructVector structVector = StructVector.empty("struct3", allocator)) { @@ -1545,9 +1549,9 @@ public void testStructOfList() { structReader.setPosition(3); FieldReader childListReader = structReader.reader("childList3"); int size = childListReader.size(); - Assert.assertEquals(1, size); + assertEquals(1, size); int data = ((List) childListReader.readObject()).get(0); - Assert.assertEquals(3, data); + assertEquals(3, data); } try (StructVector structVector = StructVector.empty("struct4", allocator)) { @@ -1564,7 +1568,7 @@ public void testStructOfList() { structReader.setPosition(3); FieldReader childListReader = structReader.reader("childList4"); int size = childListReader.size(); - Assert.assertEquals(0, size); + assertEquals(0, size); } } @@ -1618,7 +1622,7 @@ public void testMapWithNulls() { mapWriter.endMap(); writer.setValueCount(1); UnionMapReader mapReader = (UnionMapReader) new SingleStructReaderImpl(parent).reader("root"); - Assert.assertNull(mapReader.key().readInteger()); + assertNull(mapReader.key().readInteger()); assertEquals(1, mapReader.value().readInteger().intValue()); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java index 27b8f1796ee31..f17c370c89522 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestSimpleWriter.java @@ -17,6 +17,9 @@ package org.apache.arrow.vector.complex.writer; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.nio.ByteBuffer; import org.apache.arrow.memory.BufferAllocator; @@ -30,21 +33,20 @@ import org.apache.arrow.vector.complex.impl.VarBinaryWriterImpl; import org.apache.arrow.vector.complex.impl.VarCharWriterImpl; import org.apache.arrow.vector.util.Text; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestSimpleWriter { private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new RootAllocator(Integer.MAX_VALUE); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } @@ -56,7 +58,7 @@ public void testWriteByteArrayToVarBinary() throws Exception { byte[] input = new byte[] { 0x01, 0x02 }; writer.writeVarBinary(input); byte[] result = vector.get(0); - Assert.assertArrayEquals(input, result); + assertArrayEquals(input, result); } } @@ -67,7 +69,7 @@ public void testWriteByteArrayWithOffsetToVarBinary() throws Exception { byte[] input = new byte[] { 0x01, 0x02 }; writer.writeVarBinary(input, 1, 1); byte[] result = vector.get(0); - Assert.assertArrayEquals(new byte[] { 0x02 }, result); + assertArrayEquals(new byte[] { 0x02 }, result); } } @@ -79,7 +81,7 @@ public void testWriteByteBufferToVarBinary() throws Exception { ByteBuffer buffer = ByteBuffer.wrap(input); writer.writeVarBinary(buffer); byte[] result = vector.get(0); - Assert.assertArrayEquals(input, result); + assertArrayEquals(input, result); } } @@ -91,7 +93,7 @@ public void testWriteByteBufferWithOffsetToVarBinary() throws Exception { ByteBuffer buffer = ByteBuffer.wrap(input); writer.writeVarBinary(buffer, 1, 1); byte[] result = vector.get(0); - Assert.assertArrayEquals(new byte[] { 0x02 }, result); + assertArrayEquals(new byte[] { 0x02 }, result); } } @@ -102,7 +104,7 @@ public void testWriteByteArrayToLargeVarBinary() throws Exception { byte[] input = new byte[] { 0x01, 0x02 }; writer.writeLargeVarBinary(input); byte[] result = vector.get(0); - Assert.assertArrayEquals(input, result); + assertArrayEquals(input, result); } } @@ -113,7 +115,7 @@ public void testWriteByteArrayWithOffsetToLargeVarBinary() throws Exception { byte[] input = new byte[] { 0x01, 0x02 }; writer.writeLargeVarBinary(input, 1, 1); byte[] result = vector.get(0); - Assert.assertArrayEquals(new byte[] { 0x02 }, result); + assertArrayEquals(new byte[] { 0x02 }, result); } } @@ -125,7 +127,7 @@ public void testWriteByteBufferToLargeVarBinary() throws Exception { ByteBuffer buffer = ByteBuffer.wrap(input); writer.writeLargeVarBinary(buffer); byte[] result = vector.get(0); - Assert.assertArrayEquals(input, result); + assertArrayEquals(input, result); } } @@ -137,7 +139,7 @@ public void testWriteByteBufferWithOffsetToLargeVarBinary() throws Exception { ByteBuffer buffer = ByteBuffer.wrap(input); writer.writeLargeVarBinary(buffer, 1, 1); byte[] result = vector.get(0); - Assert.assertArrayEquals(new byte[] { 0x02 }, result); + assertArrayEquals(new byte[] { 0x02 }, result); } } @@ -148,7 +150,7 @@ public void testWriteStringToVarChar() throws Exception { String input = "testInput"; writer.writeVarChar(input); String result = vector.getObject(0).toString(); - Assert.assertEquals(input, result); + assertEquals(input, result); } } @@ -159,7 +161,7 @@ public void testWriteTextToVarChar() throws Exception { String input = "testInput"; writer.writeVarChar(new Text(input)); String result = vector.getObject(0).toString(); - Assert.assertEquals(input, result); + assertEquals(input, result); } } @@ -170,7 +172,7 @@ public void testWriteStringToLargeVarChar() throws Exception { String input = "testInput"; writer.writeLargeVarChar(input); String result = vector.getObject(0).toString(); - Assert.assertEquals(input, result); + assertEquals(input, result); } } @@ -181,7 +183,7 @@ public void testWriteTextToLargeVarChar() throws Exception { String input = "testInput"; writer.writeLargeVarChar(new Text(input)); String result = vector.getObject(0).toString(); - Assert.assertEquals(input, result); + assertEquals(input, result); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java index de9187edb667e..77eeb3589058d 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java @@ -18,9 +18,12 @@ package org.apache.arrow.vector.ipc; import static org.apache.arrow.vector.TestUtils.newVarCharVector; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.math.BigDecimal; @@ -84,9 +87,8 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.util.JsonStringArrayList; import org.apache.arrow.vector.util.Text; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -98,12 +100,12 @@ public class BaseFileTest { protected static final int COUNT = 10; protected BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new RootAllocator(Integer.MAX_VALUE); } - @After + @AfterEach public void tearDown() { allocator.close(); } @@ -150,17 +152,20 @@ protected void writeData(int count, StructVector parent) { protected void validateContent(int count, VectorSchemaRoot root) { for (int i = 0; i < count; i++) { - Assert.assertEquals(i, root.getVector("int").getObject(i)); - Assert.assertEquals((Short) uint1Values[i % uint1Values.length], + assertEquals(i, root.getVector("int").getObject(i)); + assertEquals((Short) uint1Values[i % uint1Values.length], ((UInt1Vector) root.getVector("uint1")).getObjectNoOverflow(i)); - Assert.assertEquals("Failed for index: " + i, (Character) uint2Values[i % uint2Values.length], - (Character) ((UInt2Vector) root.getVector("uint2")).get(i)); - Assert.assertEquals("Failed for index: " + i, (Long) uint4Values[i % uint4Values.length], - ((UInt4Vector) root.getVector("uint4")).getObjectNoOverflow(i)); - Assert.assertEquals("Failed for index: " + i, uint8Values[i % uint8Values.length], - ((UInt8Vector) root.getVector("uint8")).getObjectNoOverflow(i)); - Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getObject(i)); - Assert.assertEquals(i == 0 ? Float.NaN : i, root.getVector("float").getObject(i)); + assertEquals((Character) uint2Values[i % uint2Values.length], + (Character) ((UInt2Vector) root.getVector("uint2")).get(i), + "Failed for index: " + i); + assertEquals((Long) uint4Values[i % uint4Values.length], + ((UInt4Vector) root.getVector("uint4")).getObjectNoOverflow(i), + "Failed for index: " + i); + assertEquals(uint8Values[i % uint8Values.length], + ((UInt8Vector) root.getVector("uint8")).getObjectNoOverflow(i), + "Failed for index: " + i); + assertEquals(Long.valueOf(i), root.getVector("bigInt").getObject(i)); + assertEquals(i == 0 ? Float.NaN : i, root.getVector("float").getObject(i)); } } @@ -210,23 +215,23 @@ public void printVectors(List vectors) { } protected void validateComplexContent(int count, VectorSchemaRoot root) { - Assert.assertEquals(count, root.getRowCount()); + assertEquals(count, root.getRowCount()); printVectors(root.getFieldVectors()); for (int i = 0; i < count; i++) { Object intVal = root.getVector("int").getObject(i); if (i % 5 != 3) { - Assert.assertEquals(i, intVal); + assertEquals(i, intVal); } else { - Assert.assertNull(intVal); + assertNull(intVal); } - Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getObject(i)); - Assert.assertEquals(i % 3, ((List) root.getVector("list").getObject(i)).size()); + assertEquals(Long.valueOf(i), root.getVector("bigInt").getObject(i)); + assertEquals(i % 3, ((List) root.getVector("list").getObject(i)).size()); NullableTimeStampMilliHolder h = new NullableTimeStampMilliHolder(); FieldReader structReader = root.getVector("struct").getReader(); structReader.setPosition(i); structReader.reader("timestamp").read(h); - Assert.assertEquals(i, h.value); + assertEquals(i, h.value); } } @@ -235,7 +240,7 @@ private LocalDateTime makeDateTimeFromCount(int i) { } protected void writeDateTimeData(int count, StructVector parent) { - Assert.assertTrue(count < 100); + assertTrue(count < 100); ComplexWriter writer = new ComplexWriterImpl("root", parent); StructWriter rootWriter = writer.rootAsStruct(); DateMilliWriter dateWriter = rootWriter.dateMilli("date"); @@ -268,22 +273,22 @@ protected void writeDateTimeData(int count, StructVector parent) { } protected void validateDateTimeContent(int count, VectorSchemaRoot root) { - Assert.assertEquals(count, root.getRowCount()); + assertEquals(count, root.getRowCount()); printVectors(root.getFieldVectors()); for (int i = 0; i < count; i++) { LocalDateTime dt = makeDateTimeFromCount(i); LocalDateTime dtMilli = dt.minusNanos(i); LocalDateTime dateVal = ((DateMilliVector) root.getVector("date")).getObject(i); LocalDateTime dateExpected = dt.toLocalDate().atStartOfDay(); - Assert.assertEquals(dateExpected, dateVal); + assertEquals(dateExpected, dateVal); LocalTime timeVal = ((TimeMilliVector) root.getVector("time")).getObject(i).toLocalTime(); - Assert.assertEquals(dtMilli.toLocalTime(), timeVal); + assertEquals(dtMilli.toLocalTime(), timeVal); Object timestampMilliVal = root.getVector("timestamp-milli").getObject(i); - Assert.assertEquals(dtMilli, timestampMilliVal); + assertEquals(dtMilli, timestampMilliVal); Object timestampMilliTZVal = root.getVector("timestamp-milliTZ").getObject(i); - Assert.assertEquals(dt.atZone(ZoneId.of("Europe/Paris")).toInstant().toEpochMilli(), timestampMilliTZVal); + assertEquals(dt.atZone(ZoneId.of("Europe/Paris")).toInstant().toEpochMilli(), timestampMilliTZVal); Object timestampNanoVal = root.getVector("timestamp-nano").getObject(i); - Assert.assertEquals(dt, timestampNanoVal); + assertEquals(dt, timestampNanoVal); } } @@ -355,66 +360,66 @@ protected VectorSchemaRoot writeFlatDictionaryData( protected void validateFlatDictionary(VectorSchemaRoot root, DictionaryProvider provider) { FieldVector vector1A = root.getVector("varcharA"); - Assert.assertNotNull(vector1A); + assertNotNull(vector1A); DictionaryEncoding encoding1A = vector1A.getField().getDictionary(); - Assert.assertNotNull(encoding1A); - Assert.assertEquals(1L, encoding1A.getId()); + assertNotNull(encoding1A); + assertEquals(1L, encoding1A.getId()); - Assert.assertEquals(6, vector1A.getValueCount()); - Assert.assertEquals(0, vector1A.getObject(0)); - Assert.assertEquals(1, vector1A.getObject(1)); - Assert.assertEquals(null, vector1A.getObject(2)); - Assert.assertEquals(2, vector1A.getObject(3)); - Assert.assertEquals(1, vector1A.getObject(4)); - Assert.assertEquals(2, vector1A.getObject(5)); + assertEquals(6, vector1A.getValueCount()); + assertEquals(0, vector1A.getObject(0)); + assertEquals(1, vector1A.getObject(1)); + assertEquals(null, vector1A.getObject(2)); + assertEquals(2, vector1A.getObject(3)); + assertEquals(1, vector1A.getObject(4)); + assertEquals(2, vector1A.getObject(5)); FieldVector vector1B = root.getVector("varcharB"); - Assert.assertNotNull(vector1B); + assertNotNull(vector1B); DictionaryEncoding encoding1B = vector1A.getField().getDictionary(); - Assert.assertNotNull(encoding1B); - Assert.assertTrue(encoding1A.equals(encoding1B)); - Assert.assertEquals(1L, encoding1B.getId()); - - Assert.assertEquals(6, vector1B.getValueCount()); - Assert.assertEquals(2, vector1B.getObject(0)); - Assert.assertEquals(1, vector1B.getObject(1)); - Assert.assertEquals(2, vector1B.getObject(2)); - Assert.assertEquals(null, vector1B.getObject(3)); - Assert.assertEquals(1, vector1B.getObject(4)); - Assert.assertEquals(0, vector1B.getObject(5)); + assertNotNull(encoding1B); + assertTrue(encoding1A.equals(encoding1B)); + assertEquals(1L, encoding1B.getId()); + + assertEquals(6, vector1B.getValueCount()); + assertEquals(2, vector1B.getObject(0)); + assertEquals(1, vector1B.getObject(1)); + assertEquals(2, vector1B.getObject(2)); + assertEquals(null, vector1B.getObject(3)); + assertEquals(1, vector1B.getObject(4)); + assertEquals(0, vector1B.getObject(5)); FieldVector vector2 = root.getVector("sizes"); - Assert.assertNotNull(vector2); + assertNotNull(vector2); DictionaryEncoding encoding2 = vector2.getField().getDictionary(); - Assert.assertNotNull(encoding2); - Assert.assertEquals(2L, encoding2.getId()); + assertNotNull(encoding2); + assertEquals(2L, encoding2.getId()); - Assert.assertEquals(6, vector2.getValueCount()); - Assert.assertEquals(null, vector2.getObject(0)); - Assert.assertEquals(2, vector2.getObject(1)); - Assert.assertEquals(1, vector2.getObject(2)); - Assert.assertEquals(1, vector2.getObject(3)); - Assert.assertEquals(2, vector2.getObject(4)); - Assert.assertEquals(null, vector2.getObject(5)); + assertEquals(6, vector2.getValueCount()); + assertEquals(null, vector2.getObject(0)); + assertEquals(2, vector2.getObject(1)); + assertEquals(1, vector2.getObject(2)); + assertEquals(1, vector2.getObject(3)); + assertEquals(2, vector2.getObject(4)); + assertEquals(null, vector2.getObject(5)); Dictionary dictionary1 = provider.lookup(1L); - Assert.assertNotNull(dictionary1); + assertNotNull(dictionary1); VarCharVector dictionaryVector = ((VarCharVector) dictionary1.getVector()); - Assert.assertEquals(3, dictionaryVector.getValueCount()); - Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0)); - Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1)); - Assert.assertEquals(new Text("baz"), dictionaryVector.getObject(2)); + assertEquals(3, dictionaryVector.getValueCount()); + assertEquals(new Text("foo"), dictionaryVector.getObject(0)); + assertEquals(new Text("bar"), dictionaryVector.getObject(1)); + assertEquals(new Text("baz"), dictionaryVector.getObject(2)); Dictionary dictionary2 = provider.lookup(2L); - Assert.assertNotNull(dictionary2); + assertNotNull(dictionary2); dictionaryVector = ((VarCharVector) dictionary2.getVector()); - Assert.assertEquals(3, dictionaryVector.getValueCount()); - Assert.assertEquals(new Text("micro"), dictionaryVector.getObject(0)); - Assert.assertEquals(new Text("small"), dictionaryVector.getObject(1)); - Assert.assertEquals(new Text("large"), dictionaryVector.getObject(2)); + assertEquals(3, dictionaryVector.getValueCount()); + assertEquals(new Text("micro"), dictionaryVector.getObject(0)); + assertEquals(new Text("small"), dictionaryVector.getObject(1)); + assertEquals(new Text("large"), dictionaryVector.getObject(2)); } protected VectorSchemaRoot writeNestedDictionaryData( @@ -456,26 +461,26 @@ protected VectorSchemaRoot writeNestedDictionaryData( protected void validateNestedDictionary(VectorSchemaRoot root, DictionaryProvider provider) { FieldVector vector = root.getFieldVectors().get(0); - Assert.assertNotNull(vector); - Assert.assertNull(vector.getField().getDictionary()); + assertNotNull(vector); + assertNull(vector.getField().getDictionary()); Field nestedField = vector.getField().getChildren().get(0); DictionaryEncoding encoding = nestedField.getDictionary(); - Assert.assertNotNull(encoding); - Assert.assertEquals(2L, encoding.getId()); - Assert.assertEquals(new ArrowType.Int(32, true), encoding.getIndexType()); + assertNotNull(encoding); + assertEquals(2L, encoding.getId()); + assertEquals(new ArrowType.Int(32, true), encoding.getIndexType()); - Assert.assertEquals(3, vector.getValueCount()); - Assert.assertEquals(Arrays.asList(0, 1), vector.getObject(0)); - Assert.assertEquals(Arrays.asList(0), vector.getObject(1)); - Assert.assertEquals(Arrays.asList(1), vector.getObject(2)); + assertEquals(3, vector.getValueCount()); + assertEquals(Arrays.asList(0, 1), vector.getObject(0)); + assertEquals(Arrays.asList(0), vector.getObject(1)); + assertEquals(Arrays.asList(1), vector.getObject(2)); Dictionary dictionary = provider.lookup(2L); - Assert.assertNotNull(dictionary); + assertNotNull(dictionary); VarCharVector dictionaryVector = ((VarCharVector) dictionary.getVector()); - Assert.assertEquals(2, dictionaryVector.getValueCount()); - Assert.assertEquals(new Text("foo"), dictionaryVector.getObject(0)); - Assert.assertEquals(new Text("bar"), dictionaryVector.getObject(1)); + assertEquals(2, dictionaryVector.getValueCount()); + assertEquals(new Text("foo"), dictionaryVector.getObject(0)); + assertEquals(new Text("bar"), dictionaryVector.getObject(1)); } protected VectorSchemaRoot writeDecimalData(BufferAllocator bufferAllocator) { @@ -509,26 +514,26 @@ protected void validateDecimalData(VectorSchemaRoot root) { DecimalVector decimalVector2 = (DecimalVector) root.getVector("decimal2"); DecimalVector decimalVector3 = (DecimalVector) root.getVector("decimal3"); int count = 10; - Assert.assertEquals(count, root.getRowCount()); + assertEquals(count, root.getRowCount()); for (int i = 0; i < count; i++) { // Verify decimal 1 vector BigDecimal readValue = decimalVector1.getObject(i); ArrowType.Decimal type = (ArrowType.Decimal) decimalVector1.getField().getType(); BigDecimal genValue = new BigDecimal(BigInteger.valueOf(i), type.getScale()); - Assert.assertEquals(genValue, readValue); + assertEquals(genValue, readValue); // Verify decimal 2 vector readValue = decimalVector2.getObject(i); type = (ArrowType.Decimal) decimalVector2.getField().getType(); genValue = new BigDecimal(BigInteger.valueOf(i * (1 << 10)), type.getScale()); - Assert.assertEquals(genValue, readValue); + assertEquals(genValue, readValue); // Verify decimal 3 vector readValue = decimalVector3.getObject(i); type = (ArrowType.Decimal) decimalVector3.getField().getType(); genValue = new BigDecimal(BigInteger.valueOf(i * 1111111111111111L), type.getScale()); - Assert.assertEquals(genValue, readValue); + assertEquals(genValue, readValue); } } @@ -558,18 +563,18 @@ public void validateUnionData(int count, VectorSchemaRoot root) { unionReader.setPosition(i); switch (i % 4) { case 0: - Assert.assertEquals(i, unionReader.readInteger().intValue()); + assertEquals(i, unionReader.readInteger().intValue()); break; case 1: - Assert.assertEquals(i, unionReader.readLong().longValue()); + assertEquals(i, unionReader.readLong().longValue()); break; case 2: - Assert.assertEquals(i % 3, unionReader.size()); + assertEquals(i % 3, unionReader.size()); break; case 3: NullableTimeStampMilliHolder h = new NullableTimeStampMilliHolder(); unionReader.reader("timestamp").read(h); - Assert.assertEquals(i, h.value); + assertEquals(i, h.value); break; default: assert false : "Unexpected value in switch statement: " + i; @@ -623,7 +628,7 @@ public void writeUnionData(int count, StructVector parent) { } protected void writeVarBinaryData(int count, StructVector parent) { - Assert.assertTrue(count < 100); + assertTrue(count < 100); ComplexWriter writer = new ComplexWriterImpl("root", parent); StructWriter rootWriter = writer.rootAsStruct(); ListWriter listWriter = rootWriter.list("list"); @@ -642,7 +647,7 @@ protected void writeVarBinaryData(int count, StructVector parent) { } protected void validateVarBinary(int count, VectorSchemaRoot root) { - Assert.assertEquals(count, root.getRowCount()); + assertEquals(count, root.getRowCount()); ListVector listVector = (ListVector) root.getVector("list"); byte[] expectedArray = new byte[count]; int numVarBinaryValues = 0; @@ -650,23 +655,23 @@ protected void validateVarBinary(int count, VectorSchemaRoot root) { expectedArray[i] = (byte) i; List objList = listVector.getObject(i); if (i % 3 == 0) { - Assert.assertTrue(objList.isEmpty()); + assertTrue(objList.isEmpty()); } else { byte[] expected = Arrays.copyOfRange(expectedArray, 0, i + 1); for (int j = 0; j < i % 3; j++) { byte[] result = (byte[]) objList.get(j); - Assert.assertArrayEquals(result, expected); + assertArrayEquals(result, expected); numVarBinaryValues++; } } } // ListVector lastSet should be the index of last value + 1 - Assert.assertEquals(listVector.getLastSet(), count - 1); + assertEquals(listVector.getLastSet(), count - 1); // VarBinaryVector lastSet should be the index of last value VarBinaryVector binaryVector = (VarBinaryVector) listVector.getChildrenFromFields().get(0); - Assert.assertEquals(binaryVector.getLastSet(), numVarBinaryValues - 1); + assertEquals(binaryVector.getLastSet(), numVarBinaryValues - 1); } protected void writeBatchData(ArrowWriter writer, IntVector vector, VectorSchemaRoot root) throws IOException { @@ -762,7 +767,7 @@ protected void validateMapData(VectorSchemaRoot root) { MapVector sortedMapVector = (MapVector) root.getVector("mapSorted"); final int count = 10; - Assert.assertEquals(count, root.getRowCount()); + assertEquals(count, root.getRowCount()); UnionMapReader mapReader = new UnionMapReader(mapVector); UnionMapReader sortedMapReader = new UnionMapReader(sortedMapVector); @@ -833,7 +838,7 @@ protected void validateListAsMapData(VectorSchemaRoot root) { MapVector sortedMapVector = (MapVector) root.getVector("map"); final int count = 10; - Assert.assertEquals(count, root.getRowCount()); + assertEquals(count, root.getRowCount()); UnionMapReader sortedMapReader = new UnionMapReader(sortedMapVector); sortedMapReader.setKeyValueNames("myKey", "myValue"); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/ITTestIPCWithLargeArrowBuffers.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/ITTestIPCWithLargeArrowBuffers.java index d3c91fd144356..52d093ae29ebf 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/ITTestIPCWithLargeArrowBuffers.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/ITTestIPCWithLargeArrowBuffers.java @@ -17,10 +17,10 @@ package org.apache.arrow.vector.ipc; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; import java.io.FileInputStream; @@ -40,7 +40,7 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; -import org.junit.Test; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/MessageSerializerTest.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/MessageSerializerTest.java index 79a4b249a8a89..d5120b70d01e9 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/MessageSerializerTest.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/MessageSerializerTest.java @@ -19,10 +19,10 @@ import static java.util.Arrays.asList; import static org.apache.arrow.memory.util.LargeMemoryUtil.checkedCastToInt; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -48,9 +48,7 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; +import org.junit.jupiter.api.Test; public class MessageSerializerTest { @@ -154,9 +152,6 @@ public void testSchemaDictionaryMessageSerialization() throws IOException { assertEquals(schema, deserialized); } - @Rule - public ExpectedException expectedEx = ExpectedException.none(); - @Test public void testSerializeRecordBatchV4() throws IOException { byte[] validity = new byte[]{(byte) 255, 0}; @@ -243,5 +238,4 @@ public static void verifyBatch(ArrowRecordBatch batch, byte[] validity, byte[] v assertArrayEquals(validity, MessageSerializerTest.array(buffers.get(0))); assertArrayEquals(values, MessageSerializerTest.array(buffers.get(1))); } - } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java index 4fb5822786083..d76e5263122fe 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java @@ -19,8 +19,8 @@ import static java.nio.channels.Channels.newChannel; import static org.apache.arrow.vector.TestUtils.newVarCharVector; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -39,7 +39,7 @@ import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.types.pojo.Field; -import org.junit.Test; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFooter.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFooter.java index 38c65bddeddea..beb6500ac2ca0 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFooter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFooter.java @@ -18,7 +18,7 @@ package org.apache.arrow.vector.ipc; import static java.util.Arrays.asList; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -32,7 +32,7 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; -import org.junit.Test; +import org.junit.jupiter.api.Test; import com.google.flatbuffers.FlatBufferBuilder; diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java index 07875b25029ea..ad9ca50a14979 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderWriter.java @@ -23,11 +23,12 @@ import static org.apache.arrow.vector.TestUtils.newVarCharVector; import static org.apache.arrow.vector.TestUtils.newVector; import static org.apache.arrow.vector.testing.ValueVectorDataPopulator.setVector; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -87,10 +88,9 @@ import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel; import org.apache.arrow.vector.util.DictionaryUtility; import org.apache.arrow.vector.util.TransferPair; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestArrowReaderWriter { @@ -109,7 +109,7 @@ public class TestArrowReaderWriter { private Schema schema; private Schema encodedSchema; - @Before + @BeforeEach public void init() { allocator = new RootAllocator(Long.MAX_VALUE); @@ -150,7 +150,7 @@ public void init() { new DictionaryEncoding(/*id=*/3L, /*ordered=*/false, /*indexType=*/null)); } - @After + @AfterEach public void terminate() throws Exception { dictionaryVector1.close(); dictionaryVector2.close(); @@ -386,18 +386,17 @@ public void testWriteReadWithStructDictionaries() throws IOException { assertEquals(dictionaryVector4.getValueCount(), readDictionaryVector.getValueCount()); final BiFunction typeComparatorIgnoreName = (v1, v2) -> new TypeEqualsVisitor(v1, false, true).equals(v2); - assertTrue("Dictionary vectors are not equal", - new RangeEqualsVisitor(dictionaryVector4, readDictionaryVector, - typeComparatorIgnoreName) - .rangeEquals(new Range(0, 0, dictionaryVector4.getValueCount()))); + assertTrue(new RangeEqualsVisitor(dictionaryVector4, readDictionaryVector, typeComparatorIgnoreName) + .rangeEquals(new Range(0, 0, dictionaryVector4.getValueCount())), + "Dictionary vectors are not equal"); // Assert the decoded vector is correct try (final ValueVector readVector = DictionaryEncoder.decode(readEncoded, readDictionary)) { assertEquals(vector.getValueCount(), readVector.getValueCount()); - assertTrue("Decoded vectors are not equal", - new RangeEqualsVisitor(vector, readVector, typeComparatorIgnoreName) - .rangeEquals(new Range(0, 0, vector.getValueCount()))); + assertTrue(new RangeEqualsVisitor(vector, readVector, typeComparatorIgnoreName) + .rangeEquals(new Range(0, 0, vector.getValueCount())), + "Decoded vectors are not equal"); } } } @@ -986,7 +985,7 @@ public void testFileFooterSizeOverflow() { System.arraycopy(magicBytes, 0, data, footerOffset + 4, ArrowMagic.MAGIC_LENGTH); // test file reader - InvalidArrowFileException e = Assertions.assertThrows(InvalidArrowFileException.class, () -> { + InvalidArrowFileException e = assertThrows(InvalidArrowFileException.class, () -> { try (SeekableReadChannel channel = new SeekableReadChannel(new ByteArrayReadableSeekableByteChannel(data)); ArrowFileReader reader = new ArrowFileReader(channel, allocator)) { reader.getVectorSchemaRoot().getSchema(); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStream.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStream.java index 145bdd588e945..7f3541252772f 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStream.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStream.java @@ -17,9 +17,9 @@ package org.apache.arrow.vector.ipc; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -31,8 +31,7 @@ import org.apache.arrow.vector.TinyIntVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.types.pojo.Schema; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; public class TestArrowStream extends BaseFileTest { @Test @@ -44,15 +43,15 @@ public void testEmptyStream() throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); ArrowStreamWriter writer = new ArrowStreamWriter(root, null, out); writer.close(); - Assert.assertTrue(out.size() > 0); + assertTrue(out.size() > 0); ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); try (ArrowStreamReader reader = new ArrowStreamReader(in, allocator)) { assertEquals(schema, reader.getVectorSchemaRoot().getSchema()); // Empty should return false - Assert.assertFalse(reader.loadNextBatch()); + assertFalse(reader.loadNextBatch()); assertEquals(0, reader.getVectorSchemaRoot().getRowCount()); - Assert.assertFalse(reader.loadNextBatch()); + assertFalse(reader.loadNextBatch()); assertEquals(0, reader.getVectorSchemaRoot().getRowCount()); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStreamPipe.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStreamPipe.java index 422a63f57f7d8..4ba11fb05ff5d 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStreamPipe.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowStreamPipe.java @@ -17,8 +17,9 @@ package org.apache.arrow.vector.ipc; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import java.nio.channels.Pipe; @@ -33,8 +34,7 @@ import org.apache.arrow.vector.ipc.ArrowStreamWriter; import org.apache.arrow.vector.ipc.MessageSerializerTest; import org.apache.arrow.vector.types.pojo.Schema; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; public class TestArrowStreamPipe { Schema schema = MessageSerializerTest.testSchema(); @@ -75,7 +75,7 @@ public void run() { root.close(); } catch (IOException e) { e.printStackTrace(); - Assert.fail(e.toString()); // have to explicitly fail since we're in a separate thread + fail(e.toString()); // have to explicitly fail since we're in a separate thread } } @@ -103,14 +103,14 @@ public boolean loadNextBatch() throws IOException { return false; } VectorSchemaRoot root = getVectorSchemaRoot(); - Assert.assertEquals(16, root.getRowCount()); + assertEquals(16, root.getRowCount()); TinyIntVector vector = (TinyIntVector) root.getFieldVectors().get(0); - Assert.assertEquals((byte) (batchesRead - 1), vector.get(0)); + assertEquals((byte) (batchesRead - 1), vector.get(0)); for (int i = 1; i < 16; i++) { if (i < 8) { - Assert.assertEquals((byte) (i + 1), vector.get(i)); + assertEquals((byte) (i + 1), vector.get(i)); } else { - Assert.assertTrue(vector.isNull(i)); + assertTrue(vector.isNull(i)); } } @@ -129,7 +129,7 @@ public void run() { reader.close(); } catch (IOException e) { e.printStackTrace(); - Assert.fail(e.toString()); // have to explicitly fail since we're in a separate thread + fail(e.toString()); // have to explicitly fail since we're in a separate thread } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java index bd5bd4feabbd4..a90b97310a1cf 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestJSONFile.java @@ -17,9 +17,9 @@ package org.apache.arrow.vector.ipc; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import java.io.File; import java.io.IOException; @@ -43,8 +43,7 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.Validator; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -342,7 +341,7 @@ public void testSetStructLength() throws IOException { // initialize vectors try (VectorSchemaRoot root = reader.read();) { FieldVector vector = root.getVector("struct_nullable"); - Assert.assertEquals(7, vector.getValueCount()); + assertEquals(7, vector.getValueCount()); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestRoundTrip.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestRoundTrip.java index 5f57e90f6ba19..d1a3a6db0da44 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestRoundTrip.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestRoundTrip.java @@ -18,12 +18,13 @@ package org.apache.arrow.vector.ipc; import static org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -33,14 +34,13 @@ import java.io.IOException; import java.nio.channels.Channels; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; +import java.util.stream.Stream; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; @@ -68,55 +68,47 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; -import org.junit.AfterClass; -import org.junit.Assume; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@RunWith(Parameterized.class) public class TestRoundTrip extends BaseFileTest { private static final Logger LOGGER = LoggerFactory.getLogger(TestRoundTrip.class); private static BufferAllocator allocator; - private final String name; - private final IpcOption writeOption; - public TestRoundTrip(String name, IpcOption writeOption) { - this.name = name; - this.writeOption = writeOption; - } - - @Parameterized.Parameters(name = "options = {0}") - public static Collection getWriteOption() { + static Stream getWriteOption() { final IpcOption legacy = new IpcOption(true, MetadataVersion.V4); final IpcOption version4 = new IpcOption(false, MetadataVersion.V4); - return Arrays.asList( + return Stream.of( new Object[] {"V4Legacy", legacy}, new Object[] {"V4", version4}, new Object[] {"V5", IpcOption.DEFAULT} ); } - @BeforeClass + @BeforeAll public static void setUpClass() { allocator = new RootAllocator(Integer.MAX_VALUE); } - @AfterClass + @AfterAll public static void tearDownClass() { allocator.close(); } - @Test - public void testStruct() throws Exception { + @ParameterizedTest(name = "options = {0}") + @MethodSource("getWriteOption") + public void testStruct(String name, IpcOption writeOption) throws Exception { try (final BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, allocator.getLimit()); final StructVector parent = StructVector.empty("parent", originalVectorAllocator)) { writeData(COUNT, parent); roundTrip( + name, + writeOption, new VectorSchemaRoot(parent.getChild("root")), /* dictionaryProvider */null, TestRoundTrip::writeSingleBatch, @@ -125,13 +117,16 @@ public void testStruct() throws Exception { } } - @Test - public void testComplex() throws Exception { + @ParameterizedTest(name = "options = {0}") + @MethodSource("getWriteOption") + public void testComplex(String name, IpcOption writeOption) throws Exception { try (final BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, allocator.getLimit()); final StructVector parent = StructVector.empty("parent", originalVectorAllocator)) { writeComplexData(COUNT, parent); roundTrip( + name, + writeOption, new VectorSchemaRoot(parent.getChild("root")), /* dictionaryProvider */null, TestRoundTrip::writeSingleBatch, @@ -140,14 +135,17 @@ public void testComplex() throws Exception { } } - @Test - public void testMultipleRecordBatches() throws Exception { + @ParameterizedTest(name = "options = {0}") + @MethodSource("getWriteOption") + public void testMultipleRecordBatches(String name, IpcOption writeOption) throws Exception { int[] counts = {10, 5}; try (final BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, allocator.getLimit()); final StructVector parent = StructVector.empty("parent", originalVectorAllocator)) { writeData(counts[0], parent); roundTrip( + name, + writeOption, new VectorSchemaRoot(parent.getChild("root")), /* dictionaryProvider */null, (root, writer) -> { @@ -170,9 +168,10 @@ public void testMultipleRecordBatches() throws Exception { } } - @Test - public void testUnionV4() throws Exception { - Assume.assumeTrue(writeOption.metadataVersion == MetadataVersion.V4); + @ParameterizedTest(name = "options = {0}") + @MethodSource("getWriteOption") + public void testUnionV4(String name, IpcOption writeOption) throws Exception { + assumeTrue(writeOption.metadataVersion == MetadataVersion.V4); final File temp = File.createTempFile("arrow-test-" + name + "-", ".arrow"); temp.deleteOnExit(); final ByteArrayOutputStream memoryStream = new ByteArrayOutputStream(); @@ -188,17 +187,18 @@ public void testUnionV4() throws Exception { new ArrowStreamWriter(root, null, Channels.newChannel(memoryStream), writeOption); } }); - assertTrue(e.getMessage(), e.getMessage().contains("Cannot write union with V4 metadata")); + assertTrue(e.getMessage().contains("Cannot write union with V4 metadata"), e.getMessage()); e = assertThrows(IllegalArgumentException.class, () -> { new ArrowStreamWriter(root, null, Channels.newChannel(memoryStream), writeOption); }); - assertTrue(e.getMessage(), e.getMessage().contains("Cannot write union with V4 metadata")); + assertTrue(e.getMessage().contains("Cannot write union with V4 metadata"), e.getMessage()); } } - @Test - public void testUnionV5() throws Exception { - Assume.assumeTrue(writeOption.metadataVersion == MetadataVersion.V5); + @ParameterizedTest(name = "options = {0}") + @MethodSource("getWriteOption") + public void testUnionV5(String name, IpcOption writeOption) throws Exception { + assumeTrue(writeOption.metadataVersion == MetadataVersion.V5); try (final BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, allocator.getLimit()); final StructVector parent = StructVector.empty("parent", originalVectorAllocator)) { @@ -206,6 +206,8 @@ public void testUnionV5() throws Exception { VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root")); validateUnionData(COUNT, root); roundTrip( + name, + writeOption, root, /* dictionaryProvider */null, TestRoundTrip::writeSingleBatch, @@ -214,8 +216,9 @@ public void testUnionV5() throws Exception { } } - @Test - public void testTiny() throws Exception { + @ParameterizedTest(name = "options = {0}") + @MethodSource("getWriteOption") + public void testTiny(String name, IpcOption writeOption) throws Exception { try (final VectorSchemaRoot root = VectorSchemaRoot.create(MessageSerializerTest.testSchema(), allocator)) { root.getFieldVectors().get(0).allocateNew(); int count = 16; @@ -227,6 +230,8 @@ public void testTiny() throws Exception { root.setRowCount(count); roundTrip( + name, + writeOption, root, /* dictionaryProvider */null, TestRoundTrip::writeSingleBatch, @@ -247,8 +252,9 @@ private void validateTinyData(int count, VectorSchemaRoot root) { } } - @Test - public void testMetadata() throws Exception { + @ParameterizedTest(name = "options = {0}") + @MethodSource("getWriteOption") + public void testMetadata(String name, IpcOption writeOption) throws Exception { List childFields = new ArrayList<>(); childFields.add(new Field("varchar-child", new FieldType(true, ArrowType.Utf8.INSTANCE, null, metadata(1)), null)); childFields.add(new Field("float-child", @@ -283,6 +289,8 @@ public void testMetadata() throws Exception { } }; roundTrip( + name, + writeOption, root, /* dictionaryProvider */null, TestRoundTrip::writeSingleBatch, @@ -298,14 +306,17 @@ private Map metadata(int i) { return Collections.unmodifiableMap(map); } - @Test - public void testFlatDictionary() throws Exception { + @ParameterizedTest(name = "options = {0}") + @MethodSource("getWriteOption") + public void testFlatDictionary(String name, IpcOption writeOption) throws Exception { AtomicInteger numDictionaryBlocksWritten = new AtomicInteger(); MapDictionaryProvider provider = new MapDictionaryProvider(); try (final BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, allocator.getLimit()); final VectorSchemaRoot root = writeFlatDictionaryData(originalVectorAllocator, provider)) { roundTrip( + name, + writeOption, root, provider, (ignored, writer) -> { @@ -339,8 +350,9 @@ public void testFlatDictionary() throws Exception { } } - @Test - public void testNestedDictionary() throws Exception { + @ParameterizedTest(name = "options = {0}") + @MethodSource("getWriteOption") + public void testNestedDictionary(String name, IpcOption writeOption) throws Exception { AtomicInteger numDictionaryBlocksWritten = new AtomicInteger(); MapDictionaryProvider provider = new MapDictionaryProvider(); // data being written: @@ -356,6 +368,8 @@ public void testNestedDictionary() throws Exception { validateNestedDictionary(readRoot, streamReader); }; roundTrip( + name, + writeOption, root, provider, (ignored, writer) -> { @@ -376,8 +390,9 @@ public void testNestedDictionary() throws Exception { } } - @Test - public void testFixedSizeBinary() throws Exception { + @ParameterizedTest(name = "options = {0}") + @MethodSource("getWriteOption") + public void testFixedSizeBinary(String name, IpcOption writeOption) throws Exception { final int count = 10; final int typeWidth = 11; byte[][] byteValues = new byte[count][typeWidth]; @@ -405,6 +420,8 @@ public void testFixedSizeBinary() throws Exception { parent.setValueCount(count); roundTrip( + name, + writeOption, new VectorSchemaRoot(parent), /* dictionaryProvider */null, TestRoundTrip::writeSingleBatch, @@ -413,8 +430,9 @@ public void testFixedSizeBinary() throws Exception { } } - @Test - public void testFixedSizeList() throws Exception { + @ParameterizedTest(name = "options = {0}") + @MethodSource("getWriteOption") + public void testFixedSizeList(String name, IpcOption writeOption) throws Exception { BiConsumer validator = (expectedCount, root) -> { for (int i = 0; i < expectedCount; i++) { assertEquals(Collections2.asImmutableList(i + 0.1f, i + 10.1f), root.getVector("float-pairs") @@ -441,6 +459,8 @@ public void testFixedSizeList() throws Exception { parent.setValueCount(COUNT); roundTrip( + name, + writeOption, new VectorSchemaRoot(parent), /* dictionaryProvider */null, TestRoundTrip::writeSingleBatch, @@ -449,8 +469,9 @@ public void testFixedSizeList() throws Exception { } } - @Test - public void testVarBinary() throws Exception { + @ParameterizedTest(name = "options = {0}") + @MethodSource("getWriteOption") + public void testVarBinary(String name, IpcOption writeOption) throws Exception { try (final BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, allocator.getLimit()); final StructVector parent = StructVector.empty("parent", originalVectorAllocator)) { @@ -459,6 +480,8 @@ public void testVarBinary() throws Exception { validateVarBinary(COUNT, root); roundTrip( + name, + writeOption, root, /* dictionaryProvider */null, TestRoundTrip::writeSingleBatch, @@ -467,8 +490,9 @@ public void testVarBinary() throws Exception { } } - @Test - public void testReadWriteMultipleBatches() throws IOException { + @ParameterizedTest(name = "options = {0}") + @MethodSource("getWriteOption") + public void testReadWriteMultipleBatches(String name, IpcOption writeOption) throws IOException { File file = new File("target/mytest_nulls_multibatch.arrow"); int numBlocksWritten = 0; @@ -491,12 +515,15 @@ public void testReadWriteMultipleBatches() throws IOException { } } - @Test - public void testMap() throws Exception { + @ParameterizedTest(name = "options = {0}") + @MethodSource("getWriteOption") + public void testMap(String name, IpcOption writeOption) throws Exception { try (final BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, allocator.getLimit()); final VectorSchemaRoot root = writeMapData(originalVectorAllocator)) { roundTrip( + name, + writeOption, root, /* dictionaryProvider */null, TestRoundTrip::writeSingleBatch, @@ -505,12 +532,15 @@ public void testMap() throws Exception { } } - @Test - public void testListAsMap() throws Exception { + @ParameterizedTest(name = "options = {0}") + @MethodSource("getWriteOption") + public void testListAsMap(String name, IpcOption writeOption) throws Exception { try (final BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, allocator.getLimit()); final VectorSchemaRoot root = writeListAsMapData(originalVectorAllocator)) { roundTrip( + name, + writeOption, root, /* dictionaryProvider */null, TestRoundTrip::writeSingleBatch, @@ -539,10 +569,10 @@ private CheckedConsumer validateFileBatches( assertEquals(counts.length, recordBatches.size()); long previousOffset = 0; for (ArrowBlock rbBlock : recordBatches) { - assertTrue(rbBlock.getOffset() + " > " + previousOffset, rbBlock.getOffset() > previousOffset); + assertTrue(rbBlock.getOffset() > previousOffset, rbBlock.getOffset() + " > " + previousOffset); previousOffset = rbBlock.getOffset(); arrowReader.loadRecordBatch(rbBlock); - assertEquals("RB #" + i, counts[i], root.getRowCount()); + assertEquals(counts[i], root.getRowCount(), "RB #" + i); validator.accept(counts[i], root); try (final ArrowRecordBatch batch = unloader.getRecordBatch()) { List buffersLayout = batch.getBuffersLayout(); @@ -566,7 +596,7 @@ private CheckedConsumer validateStreamBatches( for (int n = 0; n < counts.length; n++) { assertTrue(arrowReader.loadNextBatch()); - assertEquals("RB #" + i, counts[i], root.getRowCount()); + assertEquals(counts[i], root.getRowCount(), "RB #" + i); validator.accept(counts[i], root); try (final ArrowRecordBatch batch = unloader.getRecordBatch()) { final List buffersLayout = batch.getBuffersLayout(); @@ -590,7 +620,7 @@ interface CheckedBiConsumer { void accept(T t, U u) throws Exception; } - private void roundTrip(VectorSchemaRoot root, DictionaryProvider provider, + private void roundTrip(String name, IpcOption writeOption, VectorSchemaRoot root, DictionaryProvider provider, CheckedBiConsumer writer, CheckedConsumer fileValidator, CheckedConsumer streamValidator) throws Exception { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestUIntDictionaryRoundTrip.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestUIntDictionaryRoundTrip.java index ac95121eb73f2..db1e787d04d27 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestUIntDictionaryRoundTrip.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestUIntDictionaryRoundTrip.java @@ -18,10 +18,10 @@ package org.apache.arrow.vector.ipc; import static org.apache.arrow.vector.testing.ValueVectorDataPopulator.setVector; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -29,9 +29,9 @@ import java.nio.channels.Channels; import java.nio.charset.StandardCharsets; import java.util.Arrays; -import java.util.Collection; import java.util.Map; import java.util.function.ToIntBiFunction; +import java.util.stream.Stream; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; @@ -51,41 +51,34 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; /** * Test the round-trip of dictionary encoding, * with unsigned integer as indices. */ -@RunWith(Parameterized.class) public class TestUIntDictionaryRoundTrip { - private final boolean streamMode; - - public TestUIntDictionaryRoundTrip(boolean streamMode) { - this.streamMode = streamMode; - } - private BufferAllocator allocator; private DictionaryProvider.MapDictionaryProvider dictionaryProvider; - @Before + @BeforeEach public void init() { allocator = new RootAllocator(Long.MAX_VALUE); dictionaryProvider = new DictionaryProvider.MapDictionaryProvider(); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } - private byte[] writeData(FieldVector encodedVector) throws IOException { + private byte[] writeData(boolean streamMode, FieldVector encodedVector) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); VectorSchemaRoot root = new VectorSchemaRoot( @@ -102,6 +95,7 @@ private byte[] writeData(FieldVector encodedVector) throws IOException { } private void readData( + boolean streamMode, byte[] data, Field expectedField, ToIntBiFunction valGetter, @@ -156,8 +150,9 @@ private ValueVector createEncodedVector(int bitWidth, VarCharVector dictionaryVe return field.createVector(allocator); } - @Test - public void testUInt1RoundTrip() throws IOException { + @ParameterizedTest(name = "stream mode = {0}") + @MethodSource("getRepeat") + public void testUInt1RoundTrip(boolean streamMode) throws IOException { final int vectorLength = UInt1Vector.MAX_UINT1 & UInt1Vector.PROMOTION_MASK; try (VarCharVector dictionaryVector = new VarCharVector("dictionary", allocator); UInt1Vector encodedVector1 = (UInt1Vector) createEncodedVector(8, dictionaryVector)) { @@ -170,15 +165,15 @@ public void testUInt1RoundTrip() throws IOException { } encodedVector1.setValueCount(vectorLength); setVector(dictionaryVector, dictionaryItems); - byte[] data = writeData(encodedVector1); - readData( - data, encodedVector1.getField(), (vector, index) -> (int) ((UInt1Vector) vector).getValueAsLong(index), - 8L, indices, dictionaryItems); + byte[] data = writeData(streamMode, encodedVector1); + readData(streamMode, data, encodedVector1.getField(), + (vector, index) -> (int) ((UInt1Vector) vector).getValueAsLong(index), 8L, indices, dictionaryItems); } } - @Test - public void testUInt2RoundTrip() throws IOException { + @ParameterizedTest(name = "stream mode = {0}") + @MethodSource("getRepeat") + public void testUInt2RoundTrip(boolean streamMode) throws IOException { try (VarCharVector dictionaryVector = new VarCharVector("dictionary", allocator); UInt2Vector encodedVector2 = (UInt2Vector) createEncodedVector(16, dictionaryVector)) { int[] indices = new int[]{1, 3, 5, 7, 9, UInt2Vector.MAX_UINT2}; @@ -190,14 +185,15 @@ public void testUInt2RoundTrip() throws IOException { setVector(encodedVector2, (char) 1, (char) 3, (char) 5, (char) 7, (char) 9, UInt2Vector.MAX_UINT2); setVector(dictionaryVector, dictItems); - byte[] data = writeData(encodedVector2); - readData(data, encodedVector2.getField(), (vector, index) -> (int) ((UInt2Vector) vector).getValueAsLong(index), - 16L, indices, dictItems); + byte[] data = writeData(streamMode, encodedVector2); + readData(streamMode, data, encodedVector2.getField(), + (vector, index) -> (int) ((UInt2Vector) vector).getValueAsLong(index), 16L, indices, dictItems); } } - @Test - public void testUInt4RoundTrip() throws IOException { + @ParameterizedTest(name = "stream mode = {0}") + @MethodSource("getRepeat") + public void testUInt4RoundTrip(boolean streamMode) throws IOException { final int dictLength = 10; try (VarCharVector dictionaryVector = new VarCharVector("dictionary", allocator); UInt4Vector encodedVector4 = (UInt4Vector) createEncodedVector(32, dictionaryVector)) { @@ -211,14 +207,15 @@ public void testUInt4RoundTrip() throws IOException { setVector(dictionaryVector, dictItems); setVector(encodedVector4, 1, 3, 5, 7, 9); - byte[] data = writeData(encodedVector4); - readData(data, encodedVector4.getField(), (vector, index) -> (int) ((UInt4Vector) vector).getValueAsLong(index), - 32L, indices, dictItems); + byte[] data = writeData(streamMode, encodedVector4); + readData(streamMode, data, encodedVector4.getField(), + (vector, index) -> (int) ((UInt4Vector) vector).getValueAsLong(index), 32L, indices, dictItems); } } - @Test - public void testUInt8RoundTrip() throws IOException { + @ParameterizedTest(name = "stream mode = {0}") + @MethodSource("getRepeat") + public void testUInt8RoundTrip(boolean streamMode) throws IOException { final int dictLength = 10; try (VarCharVector dictionaryVector = new VarCharVector("dictionary", allocator); UInt8Vector encodedVector8 = (UInt8Vector) createEncodedVector(64, dictionaryVector)) { @@ -231,17 +228,16 @@ public void testUInt8RoundTrip() throws IOException { setVector(encodedVector8, 1L, 3L, 5L, 7L, 9L); setVector(dictionaryVector, dictItems); - byte[] data = writeData(encodedVector8); - readData(data, encodedVector8.getField(), (vector, index) -> (int) ((UInt8Vector) vector).getValueAsLong(index), - 64L, indices, dictItems); + byte[] data = writeData(streamMode, encodedVector8); + readData(streamMode, data, encodedVector8.getField(), + (vector, index) -> (int) ((UInt8Vector) vector).getValueAsLong(index), 64L, indices, dictItems); } } - @Parameterized.Parameters(name = "stream mode = {0}") - public static Collection getRepeat() { - return Arrays.asList( - new Object[]{true}, - new Object[]{false} + static Stream getRepeat() { + return Stream.of( + Arguments.of(true), + Arguments.of(false) ); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/message/TestMessageMetadataResult.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/message/TestMessageMetadataResult.java index 0505a18484b54..89cbb9f3f1b89 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/message/TestMessageMetadataResult.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/message/TestMessageMetadataResult.java @@ -17,11 +17,11 @@ package org.apache.arrow.vector.ipc.message; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.nio.ByteBuffer; -import org.junit.Test; +import org.junit.jupiter.api.Test; public class TestMessageMetadataResult { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java index 5cc0d080053af..925f6ca254544 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java @@ -19,8 +19,8 @@ import static org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE; import static org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -46,7 +46,7 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; -import org.junit.Test; +import org.junit.jupiter.api.Test; import com.google.flatbuffers.FlatBufferBuilder; diff --git a/java/vector/src/test/java/org/apache/arrow/vector/testing/TestValueVectorPopulator.java b/java/vector/src/test/java/org/apache/arrow/vector/testing/TestValueVectorPopulator.java index 3c075c9293079..369fcc140a1b1 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/testing/TestValueVectorPopulator.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/testing/TestValueVectorPopulator.java @@ -17,8 +17,8 @@ package org.apache.arrow.vector.testing; -import static junit.framework.TestCase.assertTrue; import static org.apache.arrow.vector.testing.ValueVectorDataPopulator.setVector; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.nio.charset.StandardCharsets; @@ -56,20 +56,20 @@ import org.apache.arrow.vector.types.TimeUnit; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestValueVectorPopulator { private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new RootAllocator(Long.MAX_VALUE); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/testing/ValueVectorDataPopulator.java b/java/vector/src/test/java/org/apache/arrow/vector/testing/ValueVectorDataPopulator.java index 45e6e630792a9..66dc13d6ef545 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/testing/ValueVectorDataPopulator.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/testing/ValueVectorDataPopulator.java @@ -17,7 +17,7 @@ package org.apache.arrow.vector.testing; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.math.BigDecimal; import java.nio.charset.StandardCharsets; diff --git a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java index 872b2f3934b07..5ebfb62038919 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestExtensionType.java @@ -17,10 +17,12 @@ package org.apache.arrow.vector.types.pojo; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; import java.io.IOException; @@ -51,8 +53,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType.ExtensionType; import org.apache.arrow.vector.util.VectorBatchAppender; import org.apache.arrow.vector.validate.ValidateVectorVisitor; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; public class TestExtensionType { /** @@ -85,21 +86,19 @@ public void roundtripUuid() throws IOException { final ArrowFileReader reader = new ArrowFileReader(channel, allocator)) { reader.loadNextBatch(); final VectorSchemaRoot readerRoot = reader.getVectorSchemaRoot(); - Assert.assertEquals(root.getSchema(), readerRoot.getSchema()); + assertEquals(root.getSchema(), readerRoot.getSchema()); final Field field = readerRoot.getSchema().getFields().get(0); final UuidType expectedType = new UuidType(); - Assert.assertEquals(field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_NAME), - expectedType.extensionName()); - Assert.assertEquals(field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_METADATA), - expectedType.serialize()); + assertEquals(field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_NAME), expectedType.extensionName()); + assertEquals(field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_METADATA), expectedType.serialize()); final ExtensionTypeVector deserialized = (ExtensionTypeVector) readerRoot.getFieldVectors().get(0); - Assert.assertEquals(vector.getValueCount(), deserialized.getValueCount()); + assertEquals(vector.getValueCount(), deserialized.getValueCount()); for (int i = 0; i < vector.getValueCount(); i++) { - Assert.assertEquals(vector.isNull(i), deserialized.isNull(i)); + assertEquals(vector.isNull(i), deserialized.isNull(i)); if (!vector.isNull(i)) { - Assert.assertEquals(vector.getObject(i), deserialized.getObject(i)); + assertEquals(vector.getObject(i), deserialized.getObject(i)); } } } @@ -138,29 +137,27 @@ public void readUnderlyingType() throws IOException { final ArrowFileReader reader = new ArrowFileReader(channel, allocator)) { reader.loadNextBatch(); final VectorSchemaRoot readerRoot = reader.getVectorSchemaRoot(); - Assert.assertEquals(1, readerRoot.getSchema().getFields().size()); - Assert.assertEquals("a", readerRoot.getSchema().getFields().get(0).getName()); - Assert.assertTrue(readerRoot.getSchema().getFields().get(0).getType() instanceof ArrowType.FixedSizeBinary); - Assert.assertEquals(16, + assertEquals(1, readerRoot.getSchema().getFields().size()); + assertEquals("a", readerRoot.getSchema().getFields().get(0).getName()); + assertTrue(readerRoot.getSchema().getFields().get(0).getType() instanceof ArrowType.FixedSizeBinary); + assertEquals(16, ((ArrowType.FixedSizeBinary) readerRoot.getSchema().getFields().get(0).getType()).getByteWidth()); final Field field = readerRoot.getSchema().getFields().get(0); final UuidType expectedType = new UuidType(); - Assert.assertEquals(field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_NAME), - expectedType.extensionName()); - Assert.assertEquals(field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_METADATA), - expectedType.serialize()); + assertEquals(field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_NAME), expectedType.extensionName()); + assertEquals(field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_METADATA), expectedType.serialize()); final FixedSizeBinaryVector deserialized = (FixedSizeBinaryVector) readerRoot.getFieldVectors().get(0); - Assert.assertEquals(vector.getValueCount(), deserialized.getValueCount()); + assertEquals(vector.getValueCount(), deserialized.getValueCount()); for (int i = 0; i < vector.getValueCount(); i++) { - Assert.assertEquals(vector.isNull(i), deserialized.isNull(i)); + assertEquals(vector.isNull(i), deserialized.isNull(i)); if (!vector.isNull(i)) { final UUID uuid = vector.getObject(i); final ByteBuffer bb = ByteBuffer.allocate(16); bb.putLong(uuid.getMostSignificantBits()); bb.putLong(uuid.getLeastSignificantBits()); - Assert.assertArrayEquals(bb.array(), deserialized.get(i)); + assertArrayEquals(bb.array(), deserialized.get(i)); } } } @@ -210,26 +207,24 @@ public void roundtripLocation() throws IOException { final ArrowFileReader reader = new ArrowFileReader(channel, allocator)) { reader.loadNextBatch(); final VectorSchemaRoot readerRoot = reader.getVectorSchemaRoot(); - Assert.assertEquals(root.getSchema(), readerRoot.getSchema()); + assertEquals(root.getSchema(), readerRoot.getSchema()); final Field field = readerRoot.getSchema().getFields().get(0); final LocationType expectedType = new LocationType(); - Assert.assertEquals(field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_NAME), - expectedType.extensionName()); - Assert.assertEquals(field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_METADATA), - expectedType.serialize()); + assertEquals(field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_NAME), expectedType.extensionName()); + assertEquals(field.getMetadata().get(ExtensionType.EXTENSION_METADATA_KEY_METADATA), expectedType.serialize()); final ExtensionTypeVector deserialized = (ExtensionTypeVector) readerRoot.getFieldVectors().get(0); - Assert.assertTrue(deserialized instanceof LocationVector); - Assert.assertEquals("location", deserialized.getName()); + assertTrue(deserialized instanceof LocationVector); + assertEquals("location", deserialized.getName()); StructVector deserStruct = (StructVector) deserialized.getUnderlyingVector(); - Assert.assertNotNull(deserStruct.getChild("Latitude")); - Assert.assertNotNull(deserStruct.getChild("Longitude")); - Assert.assertEquals(vector.getValueCount(), deserialized.getValueCount()); + assertNotNull(deserStruct.getChild("Latitude")); + assertNotNull(deserStruct.getChild("Longitude")); + assertEquals(vector.getValueCount(), deserialized.getValueCount()); for (int i = 0; i < vector.getValueCount(); i++) { - Assert.assertEquals(vector.isNull(i), deserialized.isNull(i)); + assertEquals(vector.isNull(i), deserialized.isNull(i)); if (!vector.isNull(i)) { - Assert.assertEquals(vector.getObject(i), deserialized.getObject(i)); + assertEquals(vector.getObject(i), deserialized.getObject(i)); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestField.java b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestField.java index bc984fa642d52..8f98a9e9f8b53 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestField.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestField.java @@ -19,8 +19,8 @@ import static org.apache.arrow.vector.types.pojo.Schema.METADATA_KEY; import static org.apache.arrow.vector.types.pojo.Schema.METADATA_VALUE; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.util.Collections; @@ -28,7 +28,7 @@ import java.util.Map; import org.apache.arrow.vector.types.pojo.ArrowType.Int; -import org.junit.Test; +import org.junit.jupiter.api.Test; public class TestField { @@ -57,7 +57,7 @@ public void testMetadata() throws IOException { private void jsonContains(String json, String... strings) { for (String string : strings) { - assertTrue(json + " contains " + string, json.contains(string)); + assertTrue(json.contains(string), json + " contains " + string); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java index 7b62247c6e12d..e51e76737dfb7 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java @@ -20,8 +20,8 @@ import static java.util.Arrays.asList; import static org.apache.arrow.vector.types.pojo.Schema.METADATA_KEY; import static org.apache.arrow.vector.types.pojo.Schema.METADATA_VALUE; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.nio.ByteBuffer; @@ -49,7 +49,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType.Timestamp; import org.apache.arrow.vector.types.pojo.ArrowType.Union; import org.apache.arrow.vector.types.pojo.ArrowType.Utf8; -import org.junit.Test; +import org.junit.jupiter.api.Test; public class TestSchema { @@ -280,13 +280,13 @@ private void validateFieldsHashcode(java.util.List schemaFields, java.uti private void validateHashCode(Object o1, Object o2) { assertEquals(o1, o2); - assertEquals(o1 + " == " + o2, o1.hashCode(), o2.hashCode()); + assertEquals(o1.hashCode(), o2.hashCode(), o1 + " == " + o2); } private void contains(Schema schema, String... s) { String json = schema.toJson(); for (String string : s) { - assertTrue(json + " contains " + string, json.contains(string)); + assertTrue(json.contains(string), json + " contains " + string); } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/util/DecimalUtilityTest.java b/java/vector/src/test/java/org/apache/arrow/vector/util/DecimalUtilityTest.java index 804092ed94ac7..21906cb89af24 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/util/DecimalUtilityTest.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/util/DecimalUtilityTest.java @@ -17,14 +17,15 @@ package org.apache.arrow.vector.util; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.math.BigDecimal; import java.math.BigInteger; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; public class DecimalUtilityTest { private static final BigInteger[] MAX_BIG_INT = new BigInteger[]{BigInteger.valueOf(10).pow(38) @@ -45,7 +46,7 @@ public void testSetLongInDecimalArrowBuf() { DecimalUtility.writeLongToArrowBuf((long) val, buf, 0, byteLengths[x]); BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0, byteLengths[x]); BigDecimal expected = BigDecimal.valueOf(val); - Assert.assertEquals(expected, actual); + assertEquals(expected, actual); } } } @@ -64,7 +65,7 @@ public void testSetByteArrayInDecimalArrowBuf() { DecimalUtility.writeByteArrayToArrowBuf(BigInteger.valueOf(val).toByteArray(), buf, 0, byteLengths[x]); BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0, byteLengths[x]); BigDecimal expected = BigDecimal.valueOf(val); - Assert.assertEquals(expected, actual); + assertEquals(expected, actual); } long [] longValues = new long[] {Long.MIN_VALUE, 0 , Long.MAX_VALUE}; @@ -73,7 +74,7 @@ public void testSetByteArrayInDecimalArrowBuf() { DecimalUtility.writeByteArrayToArrowBuf(BigInteger.valueOf(val).toByteArray(), buf, 0, byteLengths[x]); BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0, byteLengths[x]); BigDecimal expected = BigDecimal.valueOf(val); - Assert.assertEquals(expected, actual); + assertEquals(expected, actual); } BigInteger [] decimals = new BigInteger[] {MAX_BIG_INT[x], new BigInteger("0"), MIN_BIG_INT[x]}; @@ -82,7 +83,7 @@ public void testSetByteArrayInDecimalArrowBuf() { DecimalUtility.writeByteArrayToArrowBuf(val.toByteArray(), buf, 0, byteLengths[x]); BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0, byteLengths[x]); BigDecimal expected = new BigDecimal(val); - Assert.assertEquals(expected, actual); + assertEquals(expected, actual); } } } @@ -101,7 +102,7 @@ public void testSetBigDecimalInDecimalArrowBuf() { DecimalUtility.writeBigDecimalToArrowBuf(BigDecimal.valueOf(val), buf, 0, byteLengths[x]); BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0, byteLengths[x]); BigDecimal expected = BigDecimal.valueOf(val); - Assert.assertEquals(expected, actual); + assertEquals(expected, actual); } long [] longValues = new long[] {Long.MIN_VALUE, 0 , Long.MAX_VALUE}; @@ -110,7 +111,7 @@ public void testSetBigDecimalInDecimalArrowBuf() { DecimalUtility.writeBigDecimalToArrowBuf(BigDecimal.valueOf(val), buf, 0, byteLengths[x]); BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0, byteLengths[x]); BigDecimal expected = BigDecimal.valueOf(val); - Assert.assertEquals(expected, actual); + assertEquals(expected, actual); } BigInteger [] decimals = new BigInteger[] {MAX_BIG_INT[x], new BigInteger("0"), MIN_BIG_INT[x]}; @@ -119,7 +120,7 @@ public void testSetBigDecimalInDecimalArrowBuf() { DecimalUtility.writeBigDecimalToArrowBuf(new BigDecimal(val), buf, 0, byteLengths[x]); BigDecimal actual = DecimalUtility.getBigDecimalFromArrowBuf(buf, 0, 0, byteLengths[x]); BigDecimal expected = new BigDecimal(val); - Assert.assertEquals(expected, actual); + assertEquals(expected, actual); } } } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/util/TestDataSizeRoundingUtil.java b/java/vector/src/test/java/org/apache/arrow/vector/util/TestDataSizeRoundingUtil.java index 4138ea9d7a181..636de9aab1f2b 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/util/TestDataSizeRoundingUtil.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/util/TestDataSizeRoundingUtil.java @@ -17,9 +17,9 @@ package org.apache.arrow.vector.util; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.Test; +import org.junit.jupiter.api.Test; /** * Test cases for {@link DataSizeRoundingUtil}. diff --git a/java/vector/src/test/java/org/apache/arrow/vector/util/TestElementAddressableVectorIterator.java b/java/vector/src/test/java/org/apache/arrow/vector/util/TestElementAddressableVectorIterator.java index 1c8281c85981b..fb954413e9f29 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/util/TestElementAddressableVectorIterator.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/util/TestElementAddressableVectorIterator.java @@ -17,8 +17,8 @@ package org.apache.arrow.vector.util; -import static junit.framework.TestCase.assertNull; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import java.nio.charset.StandardCharsets; @@ -27,9 +27,9 @@ import org.apache.arrow.memory.util.ArrowBufPointer; import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.VarCharVector; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; /** * Test cases for {@link ElementAddressableVectorIterator}. @@ -40,12 +40,12 @@ public class TestElementAddressableVectorIterator { private BufferAllocator allocator; - @Before + @BeforeEach public void prepare() { allocator = new RootAllocator(1024 * 1024); } - @After + @AfterEach public void shutdown() { allocator.close(); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/util/TestMapWithOrdinal.java b/java/vector/src/test/java/org/apache/arrow/vector/util/TestMapWithOrdinal.java index edd5221faf268..e0c9031c49b94 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/util/TestMapWithOrdinal.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/util/TestMapWithOrdinal.java @@ -18,20 +18,20 @@ package org.apache.arrow.vector.util; import static junit.framework.TestCase.assertNull; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Collection; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestMapWithOrdinal { private MapWithOrdinal map; - @Before + @BeforeEach public void setUp() { map = new MapWithOrdinalImpl<>(); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/util/TestMultiMapWithOrdinal.java b/java/vector/src/test/java/org/apache/arrow/vector/util/TestMultiMapWithOrdinal.java index ea829060d1c04..0c03f3dfeac46 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/util/TestMultiMapWithOrdinal.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/util/TestMultiMapWithOrdinal.java @@ -17,8 +17,11 @@ package org.apache.arrow.vector.util; -import org.junit.Assert; -import org.junit.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; public class TestMultiMapWithOrdinal { @@ -27,33 +30,33 @@ public void test() { MultiMapWithOrdinal map = new MultiMapWithOrdinal<>(); map.put("x", "1", false); - Assert.assertEquals(1, map.size()); + assertEquals(1, map.size()); map.remove("x", "1"); - Assert.assertTrue(map.isEmpty()); + assertTrue(map.isEmpty()); map.put("x", "1", false); map.put("x", "2", false); map.put("y", "0", false); - Assert.assertEquals(3, map.size()); - Assert.assertEquals(2, map.getAll("x").size()); - Assert.assertEquals("1", map.getAll("x").stream().findFirst().get()); - Assert.assertEquals("1", map.getByOrdinal(0)); - Assert.assertEquals("2", map.getByOrdinal(1)); - Assert.assertEquals("0", map.getByOrdinal(2)); - Assert.assertTrue(map.remove("x", "1")); - Assert.assertFalse(map.remove("x", "1")); - Assert.assertEquals("0", map.getByOrdinal(0)); - Assert.assertEquals(2, map.size()); + assertEquals(3, map.size()); + assertEquals(2, map.getAll("x").size()); + assertEquals("1", map.getAll("x").stream().findFirst().get()); + assertEquals("1", map.getByOrdinal(0)); + assertEquals("2", map.getByOrdinal(1)); + assertEquals("0", map.getByOrdinal(2)); + assertTrue(map.remove("x", "1")); + assertFalse(map.remove("x", "1")); + assertEquals("0", map.getByOrdinal(0)); + assertEquals(2, map.size()); map.put("x", "3", true); - Assert.assertEquals(1, map.getAll("x").size()); - Assert.assertEquals("3", map.getAll("x").stream().findFirst().get()); + assertEquals(1, map.getAll("x").size()); + assertEquals("3", map.getAll("x").stream().findFirst().get()); map.put("z", "4", false); - Assert.assertEquals(3, map.size()); + assertEquals(3, map.size()); map.put("z", "5", false); map.put("z", "6", false); - Assert.assertEquals(5, map.size()); + assertEquals(5, map.size()); map.removeAll("z"); - Assert.assertEquals(2, map.size()); - Assert.assertFalse(map.containsKey("z")); + assertEquals(2, map.size()); + assertFalse(map.containsKey("z")); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/util/TestReusableByteArray.java b/java/vector/src/test/java/org/apache/arrow/vector/util/TestReusableByteArray.java index f562e63b4bf8d..80420608c3912 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/util/TestReusableByteArray.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/util/TestReusableByteArray.java @@ -17,11 +17,11 @@ package org.apache.arrow.vector.util; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotSame; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.nio.charset.StandardCharsets; import java.util.Arrays; @@ -31,21 +31,21 @@ import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BaseValueVector; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestReusableByteArray { private BufferAllocator allocator; - @Before + @BeforeEach public void prepare() { // Permit allocating 4 vectors of max size. allocator = new RootAllocator(4 * BaseValueVector.MAX_ALLOCATION_SIZE); } - @After + @AfterEach public void shutdown() { allocator.close(); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/util/TestSchemaUtil.java b/java/vector/src/test/java/org/apache/arrow/vector/util/TestSchemaUtil.java index 52b6584086832..4375ca6e690b7 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/util/TestSchemaUtil.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/util/TestSchemaUtil.java @@ -18,7 +18,7 @@ package org.apache.arrow.vector.util; import static java.util.Arrays.asList; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; @@ -28,7 +28,7 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.SchemaUtility; -import org.junit.Test; +import org.junit.jupiter.api.Test; public class TestSchemaUtil { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/util/TestValidator.java b/java/vector/src/test/java/org/apache/arrow/vector/util/TestValidator.java index 2db70ca5d5b8d..0f72ada76f933 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/util/TestValidator.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/util/TestValidator.java @@ -18,10 +18,10 @@ package org.apache.arrow.vector.util; import static org.apache.arrow.vector.util.Validator.equalEnough; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.Test; +import org.junit.jupiter.api.Test; public class TestValidator { diff --git a/java/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java b/java/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java index 93e7535947536..45563a69ba9e6 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/util/TestVectorAppender.java @@ -17,9 +17,9 @@ package org.apache.arrow.vector.util; -import static junit.framework.TestCase.assertEquals; -import static junit.framework.TestCase.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.nio.charset.StandardCharsets; import java.util.Arrays; @@ -53,9 +53,9 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; /** * Test cases for {@link VectorAppender}. @@ -64,13 +64,13 @@ public class TestVectorAppender { private BufferAllocator allocator; - @Before + @BeforeEach public void prepare() { // Permit allocating 4 vectors of max size. allocator = new RootAllocator(4 * BaseValueVector.MAX_ALLOCATION_SIZE); } - @After + @AfterEach public void shutdown() { allocator.close(); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/util/TestVectorBatchAppender.java b/java/vector/src/test/java/org/apache/arrow/vector/util/TestVectorBatchAppender.java index 799c25c0ad71c..193736e70cadf 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/util/TestVectorBatchAppender.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/util/TestVectorBatchAppender.java @@ -17,15 +17,15 @@ package org.apache.arrow.vector.util; -import static junit.framework.TestCase.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.testing.ValueVectorDataPopulator; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; /** * Test cases for {@link VectorBatchAppender}. @@ -34,12 +34,12 @@ public class TestVectorBatchAppender { private BufferAllocator allocator; - @Before + @BeforeEach public void prepare() { allocator = new RootAllocator(1024 * 1024); } - @After + @AfterEach public void shutdown() { allocator.close(); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/util/TestVectorSchemaRootAppender.java b/java/vector/src/test/java/org/apache/arrow/vector/util/TestVectorSchemaRootAppender.java index 6309d385870c9..82a4589c3ba64 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/util/TestVectorSchemaRootAppender.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/util/TestVectorSchemaRootAppender.java @@ -17,8 +17,8 @@ package org.apache.arrow.vector.util; -import static junit.framework.TestCase.assertEquals; import static org.apache.arrow.vector.util.TestVectorAppender.assertVectorsEqual; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import org.apache.arrow.memory.BufferAllocator; @@ -28,9 +28,9 @@ import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.testing.ValueVectorDataPopulator; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; /** * Test cases for {@link VectorSchemaRootAppender}. @@ -39,12 +39,12 @@ public class TestVectorSchemaRootAppender { private BufferAllocator allocator; - @Before + @BeforeEach public void prepare() { allocator = new RootAllocator(1024 * 1024); } - @After + @AfterEach public void shutdown() { allocator.close(); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVector.java b/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVector.java index 20492036dab99..837b865c30b26 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVector.java @@ -19,8 +19,8 @@ import static org.apache.arrow.vector.testing.ValueVectorDataPopulator.setVector; import static org.apache.arrow.vector.util.ValueVectorUtility.validate; -import static org.junit.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.nio.charset.Charset; import java.util.Arrays; @@ -44,15 +44,15 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestValidateVector { private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new RootAllocator(Long.MAX_VALUE); } @@ -62,7 +62,7 @@ public void init() { private static final byte[] STR2 = "BBBBBBBBB2".getBytes(utf8Charset); private static final byte[] STR3 = "CCCC3".getBytes(utf8Charset); - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorFull.java b/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorFull.java index ca71a622bb8ea..fcf031fc33824 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorFull.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorFull.java @@ -19,9 +19,9 @@ import static org.apache.arrow.vector.testing.ValueVectorDataPopulator.setVector; import static org.apache.arrow.vector.util.ValueVectorUtility.validateFull; -import static org.junit.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.nio.charset.StandardCharsets; import java.util.Arrays; @@ -48,20 +48,20 @@ import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.FieldType; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestValidateVectorFull { private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new RootAllocator(Long.MAX_VALUE); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorSchemaRoot.java b/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorSchemaRoot.java index 1885fb21f17b6..bdb9ad3e8e530 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorSchemaRoot.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorSchemaRoot.java @@ -19,8 +19,8 @@ import static org.apache.arrow.vector.util.ValueVectorUtility.validate; import static org.apache.arrow.vector.util.ValueVectorUtility.validateFull; -import static org.junit.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; @@ -29,20 +29,20 @@ import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.testing.ValueVectorDataPopulator; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; public class TestValidateVectorSchemaRoot { private BufferAllocator allocator; - @Before + @BeforeEach public void init() { allocator = new RootAllocator(Long.MAX_VALUE); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } diff --git a/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorTypeVisitor.java b/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorTypeVisitor.java index 0ddd790d6ffab..42297e1d37fe0 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorTypeVisitor.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorTypeVisitor.java @@ -70,9 +70,9 @@ import org.apache.arrow.vector.types.Types; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.FieldType; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; /** * Test cases for {@link ValidateVectorTypeVisitor}. @@ -83,12 +83,12 @@ public class TestValidateVectorTypeVisitor { private ValidateVectorTypeVisitor visitor = new ValidateVectorTypeVisitor(); - @Before + @BeforeEach public void init() { allocator = new RootAllocator(Long.MAX_VALUE); } - @After + @AfterEach public void terminate() throws Exception { allocator.close(); } From 3999384c3e05ef8ef804ab651e1bebee8bf7670c Mon Sep 17 00:00:00 2001 From: Thomas A Caswell Date: Sat, 8 Jun 2024 16:08:01 -0400 Subject: [PATCH 85/93] MINOR: [Python] spell "language" correctly in trove classifier (#42031) ### Rationale for this change Newer (possibly unreleased) version of the Python build tools check that the classifiers are valid and the build failed due to this typo. ### What changes are included in this PR? Fix the spelling of a word ### Are these changes tested? Build will fail without these changes, has no run-time effect. ### Are there any user-facing changes? no Authored-by: Thomas A Caswell Signed-off-by: Sutou Kouhei --- python/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index f72c3a91eb436..86a90906d02f9 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -47,7 +47,7 @@ classifiers = [ 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', - 'Programming Langauge :: Python :: 3.12', + 'Programming Language :: Python :: 3.12', ] maintainers = [ {name = "Apache Arrow Developers", email = "dev@arrow.apache.org"} From 601be7687ba89f711b876397746b5f49503c0871 Mon Sep 17 00:00:00 2001 From: abandy Date: Sat, 8 Jun 2024 17:25:28 -0400 Subject: [PATCH 86/93] GH-42020: [Swift] Add Arrow decoding implementation for Swift Codable (#42023) ### Rationale for this change This change implements decode for the Arrow Swift Codable implementation. This allows the data in a RecordBatch to be copied to properties in a struct/class. The PR is a bit longer than desired but all three container types are required in order to implement the Decoder protocol. ### What changes are included in this PR? The ArrowDecoder class is included in this PR along with a class for each container type (keyed, unkeyed, and single). Most of the logic is encapsulated in the ArrowDecoder with minimal logic in each container class (Most of the methods in the container classes are a single line that calls the ArrowDecoder doDecode methods) ### Are these changes tested? Yes, a test has been added to test the three types of containers provided by the decoder. * GitHub Issue: #42020 Authored-by: Alva Bandy Signed-off-by: Sutou Kouhei --- swift/Arrow/Sources/Arrow/ArrowDecoder.swift | 347 ++++++++++++++++++ .../Arrow/Tests/ArrowTests/CodableTests.swift | 170 +++++++++ 2 files changed, 517 insertions(+) create mode 100644 swift/Arrow/Sources/Arrow/ArrowDecoder.swift create mode 100644 swift/Arrow/Tests/ArrowTests/CodableTests.swift diff --git a/swift/Arrow/Sources/Arrow/ArrowDecoder.swift b/swift/Arrow/Sources/Arrow/ArrowDecoder.swift new file mode 100644 index 0000000000000..7e0c69b1e79e8 --- /dev/null +++ b/swift/Arrow/Sources/Arrow/ArrowDecoder.swift @@ -0,0 +1,347 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation + +public class ArrowDecoder: Decoder { + var rbIndex: UInt = 0 + public var codingPath: [CodingKey] = [] + public var userInfo: [CodingUserInfoKey: Any] = [:] + public let rb: RecordBatch + public let nameToCol: [String: ArrowArrayHolder] + public let columns: [ArrowArrayHolder] + public init(_ decoder: ArrowDecoder) { + self.userInfo = decoder.userInfo + self.codingPath = decoder.codingPath + self.rb = decoder.rb + self.columns = decoder.columns + self.nameToCol = decoder.nameToCol + self.rbIndex = decoder.rbIndex + } + + public init(_ rb: RecordBatch) { + self.rb = rb + var colMapping = [String: ArrowArrayHolder]() + var columns = [ArrowArrayHolder]() + for index in 0..(_ type: T.Type) throws -> [T] { + var output = [T]() + for index in 0..(keyedBy type: Key.Type + ) -> KeyedDecodingContainer where Key: CodingKey { + let container = ArrowKeyedDecoding(self, codingPath: codingPath) + return KeyedDecodingContainer(container) + } + + public func unkeyedContainer() -> UnkeyedDecodingContainer { + return ArrowUnkeyedDecoding(self, codingPath: codingPath) + } + + public func singleValueContainer() -> SingleValueDecodingContainer { + return ArrowSingleValueDecoding(self, codingPath: codingPath) + } + + func getCol(_ name: String) throws -> AnyArray { + guard let col = self.nameToCol[name] else { + throw ArrowError.invalid("Column for key \"\(name)\" not found") + } + + guard let anyArray = col.array as? AnyArray else { + throw ArrowError.invalid("Unable to convert array to AnyArray") + } + + return anyArray + } + + func getCol(_ index: Int) throws -> AnyArray { + if index >= self.columns.count { + throw ArrowError.outOfBounds(index: Int64(index)) + } + + guard let anyArray = self.columns[index].array as? AnyArray else { + throw ArrowError.invalid("Unable to convert array to AnyArray") + } + + return anyArray + } + + func doDecode(_ key: CodingKey) throws -> T? { + let array: AnyArray = try self.getCol(key.stringValue) + return array.asAny(self.rbIndex) as? T + } + + func doDecode(_ col: Int) throws -> T? { + let array: AnyArray = try self.getCol(col) + return array.asAny(self.rbIndex) as? T + } +} + +private struct ArrowUnkeyedDecoding: UnkeyedDecodingContainer { + var codingPath: [CodingKey] + var count: Int? = 0 + var isAtEnd: Bool = false + var currentIndex: Int = 0 + let decoder: ArrowDecoder + + init(_ decoder: ArrowDecoder, codingPath: [CodingKey]) { + self.decoder = decoder + self.codingPath = codingPath + self.count = self.decoder.columns.count + } + + mutating func increment() { + self.currentIndex += 1 + self.isAtEnd = self.currentIndex >= self.count! + } + + mutating func decodeNil() throws -> Bool { + defer {increment()} + return try self.decoder.doDecode(self.currentIndex) == nil + } + + mutating func decode(_ type: T.Type) throws -> T where T: Decodable { + if type == Int8.self || type == Int16.self || + type == Int32.self || type == Int64.self || + type == UInt8.self || type == UInt16.self || + type == UInt32.self || type == UInt64.self || + type == String.self || type == Double.self || + type == Float.self || type == Date.self { + defer {increment()} + return try self.decoder.doDecode(self.currentIndex)! + } else { + throw ArrowError.invalid("Type \(type) is currently not supported") + } + } + + func nestedContainer( + keyedBy type: NestedKey.Type + ) throws -> KeyedDecodingContainer where NestedKey: CodingKey { + throw ArrowError.invalid("Nested decoding is currently not supported.") + } + + func nestedUnkeyedContainer() throws -> UnkeyedDecodingContainer { + throw ArrowError.invalid("Nested decoding is currently not supported.") + } + + func superDecoder() throws -> Decoder { + throw ArrowError.invalid("super decoding is currently not supported.") + } +} + +private struct ArrowKeyedDecoding: KeyedDecodingContainerProtocol { + var codingPath = [CodingKey]() + var allKeys = [Key]() + let decoder: ArrowDecoder + + init(_ decoder: ArrowDecoder, codingPath: [CodingKey]) { + self.decoder = decoder + self.codingPath = codingPath + } + + func contains(_ key: Key) -> Bool { + return self.decoder.nameToCol.keys.contains(key.stringValue) + } + + func decodeNil(forKey key: Key) throws -> Bool { + return try self.decoder.doDecode(key) == nil + } + + func decode(_ type: Bool.Type, forKey key: Key) throws -> Bool { + return try self.decoder.doDecode(key)! + } + + func decode(_ type: String.Type, forKey key: Key) throws -> String { + return try self.decoder.doDecode(key)! + } + + func decode(_ type: Double.Type, forKey key: Key) throws -> Double { + return try self.decoder.doDecode(key)! + } + + func decode(_ type: Float.Type, forKey key: Key) throws -> Float { + return try self.decoder.doDecode(key)! + } + + func decode(_ type: Int.Type, forKey key: Key) throws -> Int { + throw ArrowError.invalid( + "Int type is not supported (please use Int8, Int16, Int32 or Int64)") + } + + func decode(_ type: Int8.Type, forKey key: Key) throws -> Int8 { + return try self.decoder.doDecode(key)! + } + + func decode(_ type: Int16.Type, forKey key: Key) throws -> Int16 { + return try self.decoder.doDecode(key)! + } + + func decode(_ type: Int32.Type, forKey key: Key) throws -> Int32 { + return try self.decoder.doDecode(key)! + } + + func decode(_ type: Int64.Type, forKey key: Key) throws -> Int64 { + return try self.decoder.doDecode(key)! + } + + func decode(_ type: UInt.Type, forKey key: Key) throws -> UInt { + throw ArrowError.invalid( + "UInt type is not supported (please use UInt8, UInt16, UInt32 or UInt64)") + } + + func decode(_ type: UInt8.Type, forKey key: Key) throws -> UInt8 { + return try self.decoder.doDecode(key)! + } + + func decode(_ type: UInt16.Type, forKey key: Key) throws -> UInt16 { + return try self.decoder.doDecode(key)! + } + + func decode(_ type: UInt32.Type, forKey key: Key) throws -> UInt32 { + return try self.decoder.doDecode(key)! + } + + func decode(_ type: UInt64.Type, forKey key: Key) throws -> UInt64 { + return try self.decoder.doDecode(key)! + } + + func decode(_ type: T.Type, forKey key: Key) throws -> T where T: Decodable { + if type == Date.self { + return try self.decoder.doDecode(key)! + } else { + throw ArrowError.invalid("Type \(type) is currently not supported") + } + } + + func nestedContainer( + keyedBy type: NestedKey.Type, + forKey key: Key + ) throws -> KeyedDecodingContainer where NestedKey: CodingKey { + throw ArrowError.invalid("Nested decoding is currently not supported.") + } + + func nestedUnkeyedContainer(forKey key: Key) throws -> UnkeyedDecodingContainer { + throw ArrowError.invalid("Nested decoding is currently not supported.") + } + + func superDecoder() throws -> Decoder { + throw ArrowError.invalid("super decoding is currently not supported.") + } + + func superDecoder(forKey key: Key) throws -> Decoder { + throw ArrowError.invalid("super decoding is currently not supported.") + } +} + +private struct ArrowSingleValueDecoding: SingleValueDecodingContainer { + var codingPath = [CodingKey]() + let decoder: ArrowDecoder + + init(_ decoder: ArrowDecoder, codingPath: [CodingKey]) { + self.decoder = decoder + self.codingPath = codingPath + } + + func decodeNil() -> Bool { + do { + return try self.decoder.doDecode(0) == nil + } catch { + return false + } + } + + func decode(_ type: Bool.Type) throws -> Bool { + return try self.decoder.doDecode(0)! + } + + func decode(_ type: String.Type) throws -> String { + return try self.decoder.doDecode(0)! + } + + func decode(_ type: Double.Type) throws -> Double { + return try self.decoder.doDecode(0)! + } + + func decode(_ type: Float.Type) throws -> Float { + return try self.decoder.doDecode(0)! + } + + func decode(_ type: Int.Type) throws -> Int { + throw ArrowError.invalid( + "Int type is not supported (please use Int8, Int16, Int32 or Int64)") + } + + func decode(_ type: Int8.Type) throws -> Int8 { + return try self.decoder.doDecode(0)! + } + + func decode(_ type: Int16.Type) throws -> Int16 { + return try self.decoder.doDecode(0)! + } + + func decode(_ type: Int32.Type) throws -> Int32 { + return try self.decoder.doDecode(0)! + } + + func decode(_ type: Int64.Type) throws -> Int64 { + return try self.decoder.doDecode(0)! + } + + func decode(_ type: UInt.Type) throws -> UInt { + throw ArrowError.invalid( + "UInt type is not supported (please use UInt8, UInt16, UInt32 or UInt64)") + } + + func decode(_ type: UInt8.Type) throws -> UInt8 { + return try self.decoder.doDecode(0)! + } + + func decode(_ type: UInt16.Type) throws -> UInt16 { + return try self.decoder.doDecode(0)! + } + + func decode(_ type: UInt32.Type) throws -> UInt32 { + return try self.decoder.doDecode(0)! + } + + func decode(_ type: UInt64.Type) throws -> UInt64 { + return try self.decoder.doDecode(0)! + } + + func decode(_ type: T.Type) throws -> T where T: Decodable { + if type == Date.self { + return try self.decoder.doDecode(0)! + } else { + throw ArrowError.invalid("Type \(type) is currently not supported") + } + } +} diff --git a/swift/Arrow/Tests/ArrowTests/CodableTests.swift b/swift/Arrow/Tests/ArrowTests/CodableTests.swift new file mode 100644 index 0000000000000..e7359467ae1c5 --- /dev/null +++ b/swift/Arrow/Tests/ArrowTests/CodableTests.swift @@ -0,0 +1,170 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import XCTest +@testable import Arrow + +final class CodableTests: XCTestCase { + public class TestClass: Codable { + public var propBool: Bool + public var propInt8: Int8 + public var propInt16: Int16 + public var propInt32: Int32 + public var propInt64: Int64 + public var propUInt8: UInt8 + public var propUInt16: UInt16 + public var propUInt32: UInt32 + public var propUInt64: UInt64 + public var propFloat: Float + public var propDouble: Double + public var propString: String + public var propDate: Date + + public required init() { + self.propBool = false + self.propInt8 = 1 + self.propInt16 = 2 + self.propInt32 = 3 + self.propInt64 = 4 + self.propUInt8 = 5 + self.propUInt16 = 6 + self.propUInt32 = 7 + self.propUInt64 = 8 + self.propFloat = 9 + self.propDouble = 10 + self.propString = "11" + self.propDate = Date.now + } + } + + func testArrowKeyedDecoder() throws { // swiftlint:disable:this function_body_length + let date1 = Date(timeIntervalSinceReferenceDate: 86400 * 5000 + 352) + + let boolBuilder = try ArrowArrayBuilders.loadBoolArrayBuilder() + let int8Builder: NumberArrayBuilder = try ArrowArrayBuilders.loadNumberArrayBuilder() + let int16Builder: NumberArrayBuilder = try ArrowArrayBuilders.loadNumberArrayBuilder() + let int32Builder: NumberArrayBuilder = try ArrowArrayBuilders.loadNumberArrayBuilder() + let int64Builder: NumberArrayBuilder = try ArrowArrayBuilders.loadNumberArrayBuilder() + let uint8Builder: NumberArrayBuilder = try ArrowArrayBuilders.loadNumberArrayBuilder() + let uint16Builder: NumberArrayBuilder = try ArrowArrayBuilders.loadNumberArrayBuilder() + let uint32Builder: NumberArrayBuilder = try ArrowArrayBuilders.loadNumberArrayBuilder() + let uint64Builder: NumberArrayBuilder = try ArrowArrayBuilders.loadNumberArrayBuilder() + let floatBuilder: NumberArrayBuilder = try ArrowArrayBuilders.loadNumberArrayBuilder() + let doubleBuilder: NumberArrayBuilder = try ArrowArrayBuilders.loadNumberArrayBuilder() + let stringBuilder = try ArrowArrayBuilders.loadStringArrayBuilder() + let dateBuilder = try ArrowArrayBuilders.loadDate64ArrayBuilder() + + boolBuilder.append(false, true, false) + int8Builder.append(10, 11, 12) + int16Builder.append(20, 21, 22) + int32Builder.append(30, 31, 32) + int64Builder.append(40, 41, 42) + uint8Builder.append(50, 51, 52) + uint16Builder.append(60, 61, 62) + uint32Builder.append(70, 71, 72) + uint64Builder.append(80, 81, 82) + floatBuilder.append(90.1, 91.1, 92.1) + doubleBuilder.append(100.1, 101.1, 102.1) + stringBuilder.append("test0", "test1", "test2") + dateBuilder.append(date1, date1, date1) + let result = RecordBatch.Builder() + .addColumn("propBool", arrowArray: try boolBuilder.toHolder()) + .addColumn("propInt8", arrowArray: try int8Builder.toHolder()) + .addColumn("propInt16", arrowArray: try int16Builder.toHolder()) + .addColumn("propInt32", arrowArray: try int32Builder.toHolder()) + .addColumn("propInt64", arrowArray: try int64Builder.toHolder()) + .addColumn("propUInt8", arrowArray: try uint8Builder.toHolder()) + .addColumn("propUInt16", arrowArray: try uint16Builder.toHolder()) + .addColumn("propUInt32", arrowArray: try uint32Builder.toHolder()) + .addColumn("propUInt64", arrowArray: try uint64Builder.toHolder()) + .addColumn("propFloat", arrowArray: try floatBuilder.toHolder()) + .addColumn("propDouble", arrowArray: try doubleBuilder.toHolder()) + .addColumn("propString", arrowArray: try stringBuilder.toHolder()) + .addColumn("propDate", arrowArray: try dateBuilder.toHolder()) + .finish() + switch result { + case .success(let rb): + let decoder = ArrowDecoder(rb) + var testClasses = try decoder.decode(TestClass.self) + for index in 0.. = try ArrowArrayBuilders.loadNumberArrayBuilder() + int8Builder.append(10, 11, 12, nil) + let result = RecordBatch.Builder() + .addColumn("propInt8", arrowArray: try int8Builder.toHolder()) + .finish() + switch result { + case .success(let rb): + let decoder = ArrowDecoder(rb) + let testData = try decoder.decode(Int8?.self) + for index in 0.. = try ArrowArrayBuilders.loadNumberArrayBuilder() + let stringBuilder = try ArrowArrayBuilders.loadStringArrayBuilder() + int8Builder.append(10, 11, 12) + stringBuilder.append("test0", "test1", "test2") + let result = RecordBatch.Builder() + .addColumn("propInt8", arrowArray: try int8Builder.toHolder()) + .addColumn("propString", arrowArray: try stringBuilder.toHolder()) + .finish() + switch result { + case .success(let rb): + let decoder = ArrowDecoder(rb) + let testData = try decoder.decode([Int8: String].self) + var index: Int8 = 0 + for data in testData { + let str = data[10 + index] + XCTAssertEqual(str, "test\(index)") + index += 1 + } + case .failure(let err): + throw err + } + } + +} From 399408cb273c47f490f65cdad95bc184a652826c Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Sun, 9 Jun 2024 14:50:25 +0900 Subject: [PATCH 87/93] GH-42039: [Docs][Go] Fix broken link (#42040) ### Rationale for this change Fix the broken link to the correct link due to a change in the path. ### What changes are included in this PR? Updating link from the incorrect `go/` path to change in the path. - old link: https://arrow.apache.org/adbc/main/driver/go/flight_sql.html#client-options - new link: https://arrow.apache.org/adbc/main/driver/flight_sql.html#client-options ### Are these changes tested? Yes. I have checked the link. ### Are there any user-facing changes? Yes, the updated link will be visible to users. * GitHub Issue: #42039 Authored-by: Hyunseok Seo Signed-off-by: Sutou Kouhei --- go/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/README.md b/go/README.md index 4f97c49e1c7e8..220b0a230a615 100644 --- a/go/README.md +++ b/go/README.md @@ -48,7 +48,7 @@ func main() { DSN option keys are expressed as `k=v`, delimited with `;`. Some options keys are defined in ADBC, others are defined in the FlightSQL ADBC driver. -- Arrow ADBC [developer doc](https://arrow.apache.org/adbc/main/driver/go/flight_sql.html#client-options) +- Arrow ADBC [developer doc](https://arrow.apache.org/adbc/main/driver/flight_sql.html#client-options) - ADBC [source code](https://github.com/apache/arrow-adbc/blob/3d12fad1bae21029a8ff25604d6e65760c3f65bd/go/adbc/adbc.go#L149-L158) - FlightSQL driver option keys [source code](https://github.com/apache/arrow-adbc/blob/3d12fad1bae21029a8ff25604d6e65760c3f65bd/go/adbc/driver/flightsql/flightsql_adbc.go#L70-L81) From 7aaea3d9bb65ad37a17a9d3a52341f0fe2478903 Mon Sep 17 00:00:00 2001 From: abandy Date: Sun, 9 Jun 2024 19:55:16 -0400 Subject: [PATCH 88/93] GH-42041: [Swift] Fix nullable type decoder issue (#42043) ### Rationale for this change There is an issue when decoding nullable types. The previous method of checking for nil values always returned false for nullable types due too the ArrowArray types being non nullable. ### What changes are included in this PR? This PR adds a IsNull method to the ArrowDecoder to be used for null checks. Also, a check for nullable types has been added to the Unkeyed decode method. ### Are these changes tested? Yes, tests have been added/modified to test this fix. * GitHub Issue: #42041 Authored-by: Alva Bandy Signed-off-by: Sutou Kouhei --- swift/Arrow/Sources/Arrow/ArrowDecoder.swift | 31 ++++++-- .../Arrow/Tests/ArrowTests/CodableTests.swift | 73 ++++++++++++++++--- 2 files changed, 88 insertions(+), 16 deletions(-) diff --git a/swift/Arrow/Sources/Arrow/ArrowDecoder.swift b/swift/Arrow/Sources/Arrow/ArrowDecoder.swift index 7e0c69b1e79e8..9aa8a65137d28 100644 --- a/swift/Arrow/Sources/Arrow/ArrowDecoder.swift +++ b/swift/Arrow/Sources/Arrow/ArrowDecoder.swift @@ -104,6 +104,16 @@ public class ArrowDecoder: Decoder { let array: AnyArray = try self.getCol(col) return array.asAny(self.rbIndex) as? T } + + func isNull(_ key: CodingKey) throws -> Bool { + let array: AnyArray = try self.getCol(key.stringValue) + return array.asAny(self.rbIndex) == nil + } + + func isNull(_ col: Int) throws -> Bool { + let array: AnyArray = try self.getCol(col) + return array.asAny(self.rbIndex) == nil + } } private struct ArrowUnkeyedDecoding: UnkeyedDecodingContainer { @@ -126,11 +136,17 @@ private struct ArrowUnkeyedDecoding: UnkeyedDecodingContainer { mutating func decodeNil() throws -> Bool { defer {increment()} - return try self.decoder.doDecode(self.currentIndex) == nil + return try self.decoder.isNull(self.currentIndex) } mutating func decode(_ type: T.Type) throws -> T where T: Decodable { - if type == Int8.self || type == Int16.self || + if type == Int8?.self || type == Int16?.self || + type == Int32?.self || type == Int64?.self || + type == UInt8?.self || type == UInt16?.self || + type == UInt32?.self || type == UInt64?.self || + type == String?.self || type == Double?.self || + type == Float?.self || type == Date?.self || + type == Int8.self || type == Int16.self || type == Int32.self || type == Int64.self || type == UInt8.self || type == UInt16.self || type == UInt32.self || type == UInt64.self || @@ -173,7 +189,7 @@ private struct ArrowKeyedDecoding: KeyedDecodingContainerProtoco } func decodeNil(forKey key: Key) throws -> Bool { - return try self.decoder.doDecode(key) == nil + try self.decoder.isNull(key) } func decode(_ type: Bool.Type, forKey key: Key) throws -> Bool { @@ -273,7 +289,7 @@ private struct ArrowSingleValueDecoding: SingleValueDecodingContainer { func decodeNil() -> Bool { do { - return try self.decoder.doDecode(0) == nil + return try self.decoder.isNull(0) } catch { return false } @@ -338,7 +354,12 @@ private struct ArrowSingleValueDecoding: SingleValueDecodingContainer { } func decode(_ type: T.Type) throws -> T where T: Decodable { - if type == Date.self { + if type == Int8.self || type == Int16.self || + type == Int32.self || type == Int64.self || + type == UInt8.self || type == UInt16.self || + type == UInt32.self || type == UInt64.self || + type == String.self || type == Double.self || + type == Float.self || type == Date.self { return try self.decoder.doDecode(0)! } else { throw ArrowError.invalid("Type \(type) is currently not supported") diff --git a/swift/Arrow/Tests/ArrowTests/CodableTests.swift b/swift/Arrow/Tests/ArrowTests/CodableTests.swift index e7359467ae1c5..d7d3414cf6250 100644 --- a/swift/Arrow/Tests/ArrowTests/CodableTests.swift +++ b/swift/Arrow/Tests/ArrowTests/CodableTests.swift @@ -30,7 +30,7 @@ final class CodableTests: XCTestCase { public var propUInt32: UInt32 public var propUInt64: UInt64 public var propFloat: Float - public var propDouble: Double + public var propDouble: Double? public var propString: String public var propDate: Date @@ -53,7 +53,6 @@ final class CodableTests: XCTestCase { func testArrowKeyedDecoder() throws { // swiftlint:disable:this function_body_length let date1 = Date(timeIntervalSinceReferenceDate: 86400 * 5000 + 352) - let boolBuilder = try ArrowArrayBuilders.loadBoolArrayBuilder() let int8Builder: NumberArrayBuilder = try ArrowArrayBuilders.loadNumberArrayBuilder() let int16Builder: NumberArrayBuilder = try ArrowArrayBuilders.loadNumberArrayBuilder() @@ -78,7 +77,7 @@ final class CodableTests: XCTestCase { uint32Builder.append(70, 71, 72) uint64Builder.append(80, 81, 82) floatBuilder.append(90.1, 91.1, 92.1) - doubleBuilder.append(100.1, 101.1, 102.1) + doubleBuilder.append(101.1, nil, nil) stringBuilder.append("test0", "test1", "test2") dateBuilder.append(date1, date1, date1) let result = RecordBatch.Builder() @@ -102,7 +101,6 @@ final class CodableTests: XCTestCase { var testClasses = try decoder.decode(TestClass.self) for index in 0.. = try ArrowArrayBuilders.loadNumberArrayBuilder() - int8Builder.append(10, 11, 12, nil) + int8Builder.append(10, 11, 12) let result = RecordBatch.Builder() .addColumn("propInt8", arrowArray: try int8Builder.toHolder()) .finish() @@ -134,7 +136,28 @@ final class CodableTests: XCTestCase { let testData = try decoder.decode(Int8?.self) for index in 0.. = try ArrowArrayBuilders.loadNumberArrayBuilder() + int8WNilBuilder.append(10, nil, 12, nil) + let resultWNil = RecordBatch.Builder() + .addColumn("propInt8", arrowArray: try int8WNilBuilder.toHolder()) + .finish() + switch resultWNil { + case .success(let rb): + let decoder = ArrowDecoder(rb) + let testData = try decoder.decode(Int8?.self) + for index in 0.. = try ArrowArrayBuilders.loadNumberArrayBuilder() let stringBuilder = try ArrowArrayBuilders.loadStringArrayBuilder() - int8Builder.append(10, 11, 12) - stringBuilder.append("test0", "test1", "test2") + int8Builder.append(10, 11, 12, 13) + stringBuilder.append("test0", "test1", "test2", "test3") let result = RecordBatch.Builder() .addColumn("propInt8", arrowArray: try int8Builder.toHolder()) .addColumn("propString", arrowArray: try stringBuilder.toHolder()) @@ -167,4 +190,32 @@ final class CodableTests: XCTestCase { } } + func testArrowUnkeyedDecoderWithNull() throws { + let int8Builder: NumberArrayBuilder = try ArrowArrayBuilders.loadNumberArrayBuilder() + let stringWNilBuilder = try ArrowArrayBuilders.loadStringArrayBuilder() + int8Builder.append(10, 11, 12, 13) + stringWNilBuilder.append(nil, "test1", nil, "test3") + let resultWNil = RecordBatch.Builder() + .addColumn("propInt8", arrowArray: try int8Builder.toHolder()) + .addColumn("propString", arrowArray: try stringWNilBuilder.toHolder()) + .finish() + switch resultWNil { + case .success(let rb): + let decoder = ArrowDecoder(rb) + let testData = try decoder.decode([Int8: String?].self) + var index: Int8 = 0 + for data in testData { + let str = data[10 + index] + if index % 2 == 0 { + XCTAssertNil(str!) + } else { + XCTAssertEqual(str, "test\(index)") + } + index += 1 + } + case .failure(let err): + throw err + } + + } } From 7c15568aa71c1366af5eadb6140fa445f6ce4cd0 Mon Sep 17 00:00:00 2001 From: Hyunseok Seo Date: Mon, 10 Jun 2024 09:48:05 +0900 Subject: [PATCH 89/93] GH-42042: [Java] Update Unit Tests for Compressions Module (#42044) ### Rationale for this change Update package from JUnit 4(`org.junit`) to JUnit 5(`org.junit.jupiter`). ### What changes are included in this PR? - [x] Replacing `org.junit` with `org.junit.jupiter.api`. - [x] Updating `Assertions.assertXXX` to `assertXXX` using static imports - [x] Updating annotations such as `@ After`. - `@ After` -> `@ AfterEach` - [x] Doing self review ### Are these changes tested? Yes, existing tests have passed. ### Are there any user-facing changes? No. * GitHub Issue: #42042 Authored-by: Hyunseok Seo Signed-off-by: David Li --- .../TestArrowReaderWriterWithCompression.java | 59 ++++++++++--------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/java/compression/src/test/java/org/apache/arrow/compression/TestArrowReaderWriterWithCompression.java b/java/compression/src/test/java/org/apache/arrow/compression/TestArrowReaderWriterWithCompression.java index af28333746290..24d6abf3cb7c3 100644 --- a/java/compression/src/test/java/org/apache/arrow/compression/TestArrowReaderWriterWithCompression.java +++ b/java/compression/src/test/java/org/apache/arrow/compression/TestArrowReaderWriterWithCompression.java @@ -17,6 +17,11 @@ package org.apache.arrow.compression; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.channels.Channels; @@ -46,9 +51,7 @@ import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel; -import org.junit.After; -import org.junit.Assert; -import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -67,7 +70,7 @@ public void setup() { root = null; } - @After + @AfterEach public void tearDown() { if (root != null) { root.close(); @@ -134,19 +137,19 @@ public void testArrowFileZstdRoundTrip() throws Exception { try (ArrowFileReader reader = new ArrowFileReader(new ByteArrayReadableSeekableByteChannel(out.toByteArray()), allocator, CommonsCompressionFactory.INSTANCE)) { - Assertions.assertEquals(1, reader.getRecordBlocks().size()); - Assertions.assertTrue(reader.loadNextBatch()); - Assertions.assertTrue(root.equals(reader.getVectorSchemaRoot())); - Assertions.assertFalse(reader.loadNextBatch()); + assertEquals(1, reader.getRecordBlocks().size()); + assertTrue(reader.loadNextBatch()); + assertTrue(root.equals(reader.getVectorSchemaRoot())); + assertFalse(reader.loadNextBatch()); } // without compression try (ArrowFileReader reader = new ArrowFileReader(new ByteArrayReadableSeekableByteChannel(out.toByteArray()), allocator, NoCompressionCodec.Factory.INSTANCE)) { - Assertions.assertEquals(1, reader.getRecordBlocks().size()); - Exception exception = Assert.assertThrows(IllegalArgumentException.class, + assertEquals(1, reader.getRecordBlocks().size()); + Exception exception = assertThrows(IllegalArgumentException.class, reader::loadNextBatch); - Assertions.assertEquals("Please add arrow-compression module to use CommonsCompressionFactory for ZSTD", + assertEquals("Please add arrow-compression module to use CommonsCompressionFactory for ZSTD", exception.getMessage()); } } @@ -158,17 +161,17 @@ public void testArrowStreamZstdRoundTrip() throws Exception { try (ArrowStreamReader reader = new ArrowStreamReader(new ByteArrayReadableSeekableByteChannel(out.toByteArray()), allocator, CommonsCompressionFactory.INSTANCE)) { - Assert.assertTrue(reader.loadNextBatch()); - Assert.assertTrue(root.equals(reader.getVectorSchemaRoot())); - Assert.assertFalse(reader.loadNextBatch()); + assertTrue(reader.loadNextBatch()); + assertTrue(root.equals(reader.getVectorSchemaRoot())); + assertFalse(reader.loadNextBatch()); } // without compression try (ArrowStreamReader reader = new ArrowStreamReader(new ByteArrayReadableSeekableByteChannel(out.toByteArray()), allocator, NoCompressionCodec.Factory.INSTANCE)) { - Exception exception = Assert.assertThrows(IllegalArgumentException.class, + Exception exception = assertThrows(IllegalArgumentException.class, reader::loadNextBatch); - Assert.assertEquals( + assertEquals( "Please add arrow-compression module to use CommonsCompressionFactory for ZSTD", exception.getMessage() ); @@ -189,19 +192,19 @@ public void testArrowFileZstdRoundTripWithDictionary() throws Exception { try (ArrowFileReader reader = new ArrowFileReader(new ByteArrayReadableSeekableByteChannel(out.toByteArray()), allocator, CommonsCompressionFactory.INSTANCE)) { - Assertions.assertEquals(1, reader.getRecordBlocks().size()); - Assertions.assertTrue(reader.loadNextBatch()); - Assertions.assertTrue(root.equals(reader.getVectorSchemaRoot())); - Assertions.assertFalse(reader.loadNextBatch()); + assertEquals(1, reader.getRecordBlocks().size()); + assertTrue(reader.loadNextBatch()); + assertTrue(root.equals(reader.getVectorSchemaRoot())); + assertFalse(reader.loadNextBatch()); } // without compression try (ArrowFileReader reader = new ArrowFileReader(new ByteArrayReadableSeekableByteChannel(out.toByteArray()), allocator, NoCompressionCodec.Factory.INSTANCE)) { - Assertions.assertEquals(1, reader.getRecordBlocks().size()); - Exception exception = Assert.assertThrows(IllegalArgumentException.class, + assertEquals(1, reader.getRecordBlocks().size()); + Exception exception = assertThrows(IllegalArgumentException.class, reader::loadNextBatch); - Assertions.assertEquals("Please add arrow-compression module to use CommonsCompressionFactory for ZSTD", + assertEquals("Please add arrow-compression module to use CommonsCompressionFactory for ZSTD", exception.getMessage()); } dictionaryVector.close(); @@ -221,17 +224,17 @@ public void testArrowStreamZstdRoundTripWithDictionary() throws Exception { try (ArrowStreamReader reader = new ArrowStreamReader(new ByteArrayReadableSeekableByteChannel(out.toByteArray()), allocator, CommonsCompressionFactory.INSTANCE)) { - Assertions.assertTrue(reader.loadNextBatch()); - Assertions.assertTrue(root.equals(reader.getVectorSchemaRoot())); - Assertions.assertFalse(reader.loadNextBatch()); + assertTrue(reader.loadNextBatch()); + assertTrue(root.equals(reader.getVectorSchemaRoot())); + assertFalse(reader.loadNextBatch()); } // without compression try (ArrowStreamReader reader = new ArrowStreamReader(new ByteArrayReadableSeekableByteChannel(out.toByteArray()), allocator, NoCompressionCodec.Factory.INSTANCE)) { - Exception exception = Assert.assertThrows(IllegalArgumentException.class, + Exception exception = assertThrows(IllegalArgumentException.class, reader::loadNextBatch); - Assertions.assertEquals("Please add arrow-compression module to use CommonsCompressionFactory for ZSTD", + assertEquals("Please add arrow-compression module to use CommonsCompressionFactory for ZSTD", exception.getMessage()); } dictionaryVector.close(); From f086b76fdd6bd3693bf3b5c9ac89081772d61e26 Mon Sep 17 00:00:00 2001 From: Vibhatha Lakmal Abeykoon Date: Mon, 10 Jun 2024 06:26:39 +0530 Subject: [PATCH 90/93] GH-40819: [Java] Adding Spotless to Algorithm module (#41825) ### Rationale for this change Adding code style and formatting options for Algorithm module. ### What changes are included in this PR? Code formatting spotless plugin has been added. ### Are these changes tested? Yes, but doesn't involve test cases, the plugin itself corrects. ### Are there any user-facing changes? No * GitHub Issue: #40819 Lead-authored-by: Vibhatha Abeykoon Co-authored-by: Vibhatha Lakmal Abeykoon Co-authored-by: David Li Signed-off-by: David Li --- .gitignore | 4 +- docs/source/developers/java/development.rst | 46 ++- {.mvn => java/.mvn}/develocity.xml | 0 {.mvn => java/.mvn}/extensions.xml | 0 java/algorithm/pom.xml | 7 +- .../deduplicate/DeduplicationUtils.java | 16 +- .../deduplicate/VectorRunDeduplicator.java | 27 +- .../dictionary/DictionaryBuilder.java | 30 +- .../dictionary/DictionaryEncoder.java | 11 +- .../HashTableBasedDictionaryBuilder.java | 34 +-- .../HashTableDictionaryEncoder.java | 69 ++--- .../dictionary/LinearDictionaryEncoder.java | 44 +-- .../dictionary/SearchDictionaryEncoder.java | 41 ++- .../SearchTreeBasedDictionaryBuilder.java | 46 ++- .../arrow/algorithm/misc/PartialSumUtils.java | 41 ++- .../arrow/algorithm/rank/VectorRank.java | 15 +- .../algorithm/search/ParallelSearcher.java | 187 ++++++------ .../algorithm/search/VectorRangeSearcher.java | 213 +++++++------ .../algorithm/search/VectorSearcher.java | 26 +- .../sort/CompositeVectorComparator.java | 17 +- .../sort/DefaultVectorComparators.java | 126 ++++---- .../sort/FixedWidthInPlaceVectorSorter.java | 25 +- .../FixedWidthOutOfPlaceVectorSorter.java | 35 ++- .../sort/GeneralOutOfPlaceVectorSorter.java | 20 +- .../algorithm/sort/InPlaceVectorSorter.java | 7 +- .../arrow/algorithm/sort/IndexSorter.java | 33 +- .../arrow/algorithm/sort/InsertionSorter.java | 23 +- .../arrow/algorithm/sort/OffHeapIntStack.java | 5 +- .../sort/OutOfPlaceVectorSorter.java | 8 +- .../sort/StableVectorComparator.java | 13 +- .../VariableWidthOutOfPlaceVectorSorter.java | 56 ++-- .../algorithm/sort/VectorValueComparator.java | 56 ++-- .../deduplicate/TestDeduplicationUtils.java | 46 +-- .../TestVectorRunDeduplicator.java | 19 +- .../TestHashTableBasedDictionaryBuilder.java | 62 ++-- .../TestHashTableDictionaryEncoder.java | 72 +++-- .../TestLinearDictionaryEncoder.java | 72 +++-- .../TestSearchDictionaryEncoder.java | 84 ++--- .../TestSearchTreeBasedDictionaryBuilder.java | 90 ++++-- .../algorithm/misc/TestPartialSumUtils.java | 18 +- .../arrow/algorithm/rank/TestVectorRank.java | 20 +- .../search/TestParallelSearcher.java | 36 ++- .../search/TestVectorRangeSearcher.java | 30 +- .../algorithm/search/TestVectorSearcher.java | 30 +- .../sort/TestCompositeVectorComparator.java | 18 +- .../sort/TestDefaultVectorComparator.java | 167 ++++++---- .../TestFixedWidthInPlaceVectorSorter.java | 48 ++- .../TestFixedWidthOutOfPlaceVectorSorter.java | 69 +++-- .../algorithm/sort/TestFixedWidthSorting.java | 126 +++++--- .../TestGeneralOutOfPlaceVectorSorter.java | 79 ++--- .../arrow/algorithm/sort/TestIndexSorter.java | 31 +- .../algorithm/sort/TestInsertionSorter.java | 9 +- .../algorithm/sort/TestOffHeapIntStack.java | 5 +- .../sort/TestOutOfPlaceVectorSorter.java | 6 +- .../arrow/algorithm/sort/TestSortingUtil.java | 136 +++++---- .../sort/TestStableVectorComparator.java | 50 +-- ...stVariableWidthOutOfPlaceVectorSorter.java | 40 +-- .../sort/TestVariableWidthSorting.java | 44 +-- java/dev/checkstyle/checkstyle-spotless.xml | 286 ++++++++++++++++++ .../asf-java.license} | 0 java/dev/license/asf-xml.license | 11 + java/maven/pom.xml | 2 +- java/pom.xml | 22 +- 63 files changed, 1716 insertions(+), 1293 deletions(-) rename {.mvn => java/.mvn}/develocity.xml (100%) rename {.mvn => java/.mvn}/extensions.xml (100%) create mode 100644 java/dev/checkstyle/checkstyle-spotless.xml rename java/dev/{checkstyle/checkstyle.license => license/asf-java.license} (100%) create mode 100644 java/dev/license/asf-xml.license diff --git a/.gitignore b/.gitignore index 3192069d1ac7a..52ffa6c6124c2 100644 --- a/.gitignore +++ b/.gitignore @@ -102,8 +102,8 @@ __debug_bin .envrc # Develocity -.mvn/.gradle-enterprise/ -.mvn/.develocity/ +java/.mvn/.gradle-enterprise/ +java/.mvn/.develocity/ # rat filtered_rat.txt diff --git a/docs/source/developers/java/development.rst b/docs/source/developers/java/development.rst index 9f78eccf6c525..dd1839257a30e 100644 --- a/docs/source/developers/java/development.rst +++ b/docs/source/developers/java/development.rst @@ -110,7 +110,46 @@ integration tests, you would do: Code Style ========== -Java code style is enforced with Checkstyle. The configuration is located at `checkstyle`_. +The current Java code follows the `Google Java Style`_ with Apache license headers. + +Java code style is checked by `Spotless`_ during the build, and the continuous integration build will verify +that changes adhere to the style guide. + +Automatically fixing code style issues +-------------------------------------- + +- You can check the style without building the project with ``mvn spotless:check``. +- You can autoformat the source with ``mvn spotless:apply``. + +Example: + +.. code-block:: bash + + The following files had format violations: + src/main/java/org/apache/arrow/algorithm/rank/VectorRank.java + @@ -15,7 +15,6 @@ + ·*·limitations·under·the·License. + ·*/ + + - + package·org.apache.arrow.algorithm.rank; + + import·java.util.stream.IntStream; + Run 'mvn spotless:apply' to fix these violations. + +Code Formatter for Intellij IDEA and Eclipse +-------------------------------------------- + +Follow the instructions to set up google-java-format for: + +- `Eclipse`_ +- `IntelliJ`_ + + +Checkstyle +---------- + +Checkstyle is also used for general linting. The configuration is located at `checkstyle`_. You can also just check the style without building the project. This checks the code style of all source code under the current directory or from within an individual module. @@ -137,7 +176,10 @@ This applies the style to all pom.xml files under the current directory or from .. _conbench: https://github.com/conbench/conbench .. _checkstyle: https://github.com/apache/arrow/blob/main/java/dev/checkstyle/checkstyle.xml .. _Apache Maven pom.xml guidelines: https://maven.apache.org/developers/conventions/code.html#pom-code-convention - +.. _Spotless: https://github.com/diffplug/spotless +.. _Google Java Style: https://google.github.io/styleguide/javaguide.html +.. _Eclipse: https://github.com/google/google-java-format?tab=readme-ov-file#eclipse +.. _IntelliJ: https://github.com/google/google-java-format?tab=readme-ov-file#intellij-android-studio-and-other-jetbrains-ides Build Caching ============= diff --git a/.mvn/develocity.xml b/java/.mvn/develocity.xml similarity index 100% rename from .mvn/develocity.xml rename to java/.mvn/develocity.xml diff --git a/.mvn/extensions.xml b/java/.mvn/extensions.xml similarity index 100% rename from .mvn/extensions.xml rename to java/.mvn/extensions.xml diff --git a/java/algorithm/pom.xml b/java/algorithm/pom.xml index 0854da48b718a..5984cce766d9e 100644 --- a/java/algorithm/pom.xml +++ b/java/algorithm/pom.xml @@ -20,6 +20,11 @@ Arrow Algorithms (Experimental/Contrib) A collection of algorithms for working with ValueVectors. + + dev/checkstyle/checkstyle-spotless.xml + none + + org.apache.arrow @@ -47,6 +52,4 @@ value-annotations - - diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/deduplicate/DeduplicationUtils.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/deduplicate/DeduplicationUtils.java index 8811e43d3d08d..e9364b2a85b7b 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/deduplicate/DeduplicationUtils.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/deduplicate/DeduplicationUtils.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.deduplicate; import org.apache.arrow.memory.ArrowBuf; @@ -26,18 +25,18 @@ import org.apache.arrow.vector.compare.RangeEqualsVisitor; import org.apache.arrow.vector.util.DataSizeRoundingUtil; -/** - * Utilities for vector deduplication. - */ +/** Utilities for vector deduplication. */ class DeduplicationUtils { /** * Gets the start positions of the first distinct values in a vector. + * * @param vector the target vector. * @param runStarts the bit set to hold the start positions. * @param vector type. */ - public static void populateRunStartIndicators(V vector, ArrowBuf runStarts) { + public static void populateRunStartIndicators( + V vector, ArrowBuf runStarts) { int bufSize = DataSizeRoundingUtil.divideBy8Ceil(vector.getValueCount()); Preconditions.checkArgument(runStarts.capacity() >= bufSize); runStarts.setZero(0, bufSize); @@ -55,6 +54,7 @@ public static void populateRunStartIndicators(V vector, /** * Gets the run lengths, given the start positions. + * * @param runStarts the bit set for start positions. * @param runLengths the run length vector to populate. * @param valueCount the number of values in the bit set. @@ -76,15 +76,15 @@ public static void populateRunLengths(ArrowBuf runStarts, IntVector runLengths, } /** - * Gets distinct values from the input vector by removing adjacent - * duplicated values. + * Gets distinct values from the input vector by removing adjacent duplicated values. + * * @param indicators the bit set containing the start positions of distinct values. * @param inputVector the input vector. * @param outputVector the output vector. * @param vector type. */ public static void populateDeduplicatedValues( - ArrowBuf indicators, V inputVector, V outputVector) { + ArrowBuf indicators, V inputVector, V outputVector) { int dstIdx = 0; for (int srcIdx = 0; srcIdx < inputVector.getValueCount(); srcIdx++) { if (BitVectorHelper.get(indicators, srcIdx) != 0) { diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/deduplicate/VectorRunDeduplicator.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/deduplicate/VectorRunDeduplicator.java index 5ef03cbe4a734..4e49de14f5956 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/deduplicate/VectorRunDeduplicator.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/deduplicate/VectorRunDeduplicator.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.deduplicate; import org.apache.arrow.memory.ArrowBuf; @@ -26,29 +25,28 @@ import org.apache.arrow.vector.util.DataSizeRoundingUtil; /** - * Remove adjacent equal elements from a vector. - * If the vector is sorted, it removes all duplicated values in the vector. + * Remove adjacent equal elements from a vector. If the vector is sorted, it removes all duplicated + * values in the vector. + * * @param vector type. */ public class VectorRunDeduplicator implements AutoCloseable { /** - * Bit set for distinct values. - * If the value at some index is not equal to the previous value, - * its bit is set to 1, otherwise its bit is set to 0. + * Bit set for distinct values. If the value at some index is not equal to the previous value, its + * bit is set to 1, otherwise its bit is set to 0. */ private ArrowBuf distinctValueBuffer; - /** - * The vector to deduplicate. - */ + /** The vector to deduplicate. */ private final V vector; private final BufferAllocator allocator; /** * Constructs a vector run deduplicator for a given vector. - * @param vector the vector to deduplicate. Ownership is NOT taken. + * + * @param vector the vector to deduplicate. Ownership is NOT taken. * @param allocator the allocator used for allocating buffers for start indices. */ public VectorRunDeduplicator(V vector, BufferAllocator allocator) { @@ -65,17 +63,20 @@ private void createDistinctValueBuffer() { /** * Gets the number of values which are different from their predecessor. + * * @return the run count. */ public int getRunCount() { if (distinctValueBuffer == null) { createDistinctValueBuffer(); } - return vector.getValueCount() - BitVectorHelper.getNullCount(distinctValueBuffer, vector.getValueCount()); + return vector.getValueCount() + - BitVectorHelper.getNullCount(distinctValueBuffer, vector.getValueCount()); } /** * Gets the vector with deduplicated adjacent values removed. + * * @param outVector the output vector. */ public void populateDeduplicatedValues(V outVector) { @@ -88,6 +89,7 @@ public void populateDeduplicatedValues(V outVector) { /** * Gets the length of each distinct value. + * * @param lengthVector the vector for holding length values. */ public void populateRunLengths(IntVector lengthVector) { @@ -95,7 +97,8 @@ public void populateRunLengths(IntVector lengthVector) { createDistinctValueBuffer(); } - DeduplicationUtils.populateRunLengths(distinctValueBuffer, lengthVector, vector.getValueCount()); + DeduplicationUtils.populateRunLengths( + distinctValueBuffer, lengthVector, vector.getValueCount()); } @Override diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/dictionary/DictionaryBuilder.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/dictionary/DictionaryBuilder.java index 398368d1fc612..88c4e4dc65450 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/dictionary/DictionaryBuilder.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/dictionary/DictionaryBuilder.java @@ -14,33 +14,31 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.dictionary; import org.apache.arrow.vector.ValueVector; /** - * A dictionary builder is intended for the scenario frequently encountered in practice: - * the dictionary is not known a priori, so it is generated dynamically. - * In particular, when a new value arrives, it is tested to check if it is already - * in the dictionary. If so, it is simply neglected, otherwise, it is added to the dictionary. - *

- * The dictionary builder is intended to build a single dictionary. - * So it cannot be used for different dictionaries. - *

+ * A dictionary builder is intended for the scenario frequently encountered in practice: the + * dictionary is not known a priori, so it is generated dynamically. In particular, when a new value + * arrives, it is tested to check if it is already in the dictionary. If so, it is simply neglected, + * otherwise, it is added to the dictionary. + * + *

The dictionary builder is intended to build a single dictionary. So it cannot be used for + * different dictionaries. + * *

Below gives the sample code for using the dictionary builder + * * 

{@code
  * DictionaryBuilder dictionaryBuilder = ...
  * ...
  * dictionaryBuild.addValue(newValue);
  * ...
  * }
- *

- *

- * With the above code, the dictionary vector will be populated, - * and it can be retrieved by the {@link DictionaryBuilder#getDictionary()} method. - * After that, dictionary encoding can proceed with the populated dictionary.. - *

+ * + *

With the above code, the dictionary vector will be populated, and it can be retrieved by the + * {@link DictionaryBuilder#getDictionary()} method. After that, dictionary encoding can proceed + * with the populated dictionary.. * * @param the dictionary vector type. */ @@ -58,7 +56,7 @@ public interface DictionaryBuilder { * Try to add an element from the target vector to the dictionary. * * @param targetVector the target vector containing new element. - * @param targetIndex the index of the new element in the target vector. + * @param targetIndex the index of the new element in the target vector. * @return the index of the new element in the dictionary. */ int addValue(V targetVector, int targetIndex); diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/dictionary/DictionaryEncoder.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/dictionary/DictionaryEncoder.java index cda7b3bf9540e..16e27c3a23e72 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/dictionary/DictionaryEncoder.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/dictionary/DictionaryEncoder.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.dictionary; import org.apache.arrow.vector.BaseIntVector; @@ -22,8 +21,9 @@ /** * A dictionary encoder translates one vector into another one based on a dictionary vector. - * According to Arrow specification, the encoded vector must be an integer based vector, which - * is the index of the original vector element in the dictionary. + * According to Arrow specification, the encoded vector must be an integer based vector, which is + * the index of the original vector element in the dictionary. + * * @param type of the encoded vector. * @param type of the vector to encode. It is also the type of the dictionary vector. */ @@ -31,9 +31,10 @@ public interface DictionaryEncoder the dictionary vector type. */ -public class HashTableBasedDictionaryBuilder implements DictionaryBuilder { +public class HashTableBasedDictionaryBuilder + implements DictionaryBuilder { - /** - * The dictionary to be built. - */ + /** The dictionary to be built. */ private final V dictionary; - /** - * If null should be encoded. - */ + /** If null should be encoded. */ private final boolean encodeNull; /** - * The hash map for distinct dictionary entries. - * The key is the pointer to the dictionary element, whereas the value is the index in the dictionary. + * The hash map for distinct dictionary entries. The key is the pointer to the dictionary element, + * whereas the value is the index in the dictionary. */ private HashMap hashMap = new HashMap<>(); - /** - * The hasher used for calculating the hash code. - */ + /** The hasher used for calculating the hash code. */ private final ArrowBufHasher hasher; - /** - * Next pointer to try to add to the hash table. - */ + /** Next pointer to try to add to the hash table. */ private ArrowBufPointer nextPointer; /** @@ -83,7 +73,7 @@ public HashTableBasedDictionaryBuilder(V dictionary, boolean encodeNull) { * * @param dictionary the dictionary to populate. * @param encodeNull if null values should be added to the dictionary. - * @param hasher the hasher used to compute the hash code. + * @param hasher the hasher used to compute the hash code. */ public HashTableBasedDictionaryBuilder(V dictionary, boolean encodeNull, ArrowBufHasher hasher) { this.dictionary = dictionary; @@ -125,7 +115,7 @@ public int addValues(V targetVector) { * Try to add an element from the target vector to the dictionary. * * @param targetVector the target vector containing new element. - * @param targetIndex the index of the new element in the target vector. + * @param targetIndex the index of the new element in the target vector. * @return the index of the new element in the dictionary. */ @Override diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/dictionary/HashTableDictionaryEncoder.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/dictionary/HashTableDictionaryEncoder.java index bea1a784c3d6a..ac7a7d32bf597 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/dictionary/HashTableDictionaryEncoder.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/dictionary/HashTableDictionaryEncoder.java @@ -14,11 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.dictionary; import java.util.HashMap; - import org.apache.arrow.memory.util.ArrowBufPointer; import org.apache.arrow.memory.util.hash.ArrowBufHasher; import org.apache.arrow.memory.util.hash.SimpleHasher; @@ -27,43 +25,35 @@ /** * Dictionary encoder based on hash table. + * * @param encoded vector type. * @param decoded vector type, which is also the dictionary type. */ public class HashTableDictionaryEncoder implements DictionaryEncoder { - /** - * The dictionary for encoding/decoding. - * It must be sorted. - */ + /** The dictionary for encoding/decoding. It must be sorted. */ private final D dictionary; - /** - * The hasher used to compute the hash code. - */ + /** The hasher used to compute the hash code. */ private final ArrowBufHasher hasher; - /** - * A flag indicating if null should be encoded. - */ + /** A flag indicating if null should be encoded. */ private final boolean encodeNull; /** - * The hash map for distinct dictionary entries. - * The key is the pointer to the dictionary element, whereas the value is the index in the dictionary. + * The hash map for distinct dictionary entries. The key is the pointer to the dictionary element, + * whereas the value is the index in the dictionary. */ private HashMap hashMap = new HashMap<>(); - /** - * The pointer used to probe each element to encode. - */ + /** The pointer used to probe each element to encode. */ private ArrowBufPointer reusablePointer; /** * Constructs a dictionary encoder. - * @param dictionary the dictionary. * + * @param dictionary the dictionary. */ public HashTableDictionaryEncoder(D dictionary) { this(dictionary, false); @@ -71,20 +61,17 @@ public HashTableDictionaryEncoder(D dictionary) { /** * Constructs a dictionary encoder. + * * @param dictionary the dictionary. - * @param encodeNull a flag indicating if null should be encoded. - * It determines the behaviors for processing null values in the input during encoding/decoding. - *

  • - * For encoding, when a null is encountered in the input, - * 1) If the flag is set to true, the encoder searches for the value in the dictionary, - * and outputs the index in the dictionary. - * 2) If the flag is set to false, the encoder simply produces a null in the output. - *
    • - *
  • - * For decoding, when a null is encountered in the input, - * 1) If the flag is set to true, the decoder should never expect a null in the input. - * 2) If set to false, the decoder simply produces a null in the output. - *
    • + * @param encodeNull a flag indicating if null should be encoded. It determines the behaviors for + * processing null values in the input during encoding/decoding. + *
  • For encoding, when a null is encountered in the input, 1) If the flag is set to true, + * the encoder searches for the value in the dictionary, and outputs the index in the + * dictionary. 2) If the flag is set to false, the encoder simply produces a null in the + * output. + *
  • For decoding, when a null is encountered in the input, 1) If the flag is set to true, + * the decoder should never expect a null in the input. 2) If set to false, the decoder + * simply produces a null in the output. */ public HashTableDictionaryEncoder(D dictionary, boolean encodeNull) { this(dictionary, encodeNull, SimpleHasher.INSTANCE); @@ -92,13 +79,13 @@ public HashTableDictionaryEncoder(D dictionary, boolean encodeNull) { /** * Constructs a dictionary encoder. + * * @param dictionary the dictionary. - * @param encodeNull a flag indicating if null should be encoded. - * It determines the behaviors for processing null values in the input during encoding. - * When a null is encountered in the input, - * 1) If the flag is set to true, the encoder searches for the value in the dictionary, - * and outputs the index in the dictionary. - * 2) If the flag is set to false, the encoder simply produces a null in the output. + * @param encodeNull a flag indicating if null should be encoded. It determines the behaviors for + * processing null values in the input during encoding. When a null is encountered in the + * input, 1) If the flag is set to true, the encoder searches for the value in the dictionary, + * and outputs the index in the dictionary. 2) If the flag is set to false, the encoder simply + * produces a null in the output. * @param hasher the hasher used to calculate the hash code. */ public HashTableDictionaryEncoder(D dictionary, boolean encodeNull, ArrowBufHasher hasher) { @@ -120,12 +107,12 @@ private void buildHashMap() { } /** - * Encodes an input vector by a hash table. - * So the algorithm takes O(n) time, where n is the length of the input vector. + * Encodes an input vector by a hash table. So the algorithm takes O(n) time, where n is the + * length of the input vector. * - * @param input the input vector. + * @param input the input vector. * @param output the output vector. - **/ + */ @Override public void encode(D input, E output) { for (int i = 0; i < input.getValueCount(); i++) { diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/dictionary/LinearDictionaryEncoder.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/dictionary/LinearDictionaryEncoder.java index 84a3a96af8ef1..9aeff22005751 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/dictionary/LinearDictionaryEncoder.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/dictionary/LinearDictionaryEncoder.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.dictionary; import org.apache.arrow.vector.BaseIntVector; @@ -24,20 +23,17 @@ /** * Dictionary encoder based on linear search. + * * @param encoded vector type. * @param decoded vector type, which is also the dictionary type. */ public class LinearDictionaryEncoder implements DictionaryEncoder { - /** - * The dictionary for encoding. - */ + /** The dictionary for encoding. */ private final D dictionary; - /** - * A flag indicating if null should be encoded. - */ + /** A flag indicating if null should be encoded. */ private final boolean encodeNull; private RangeEqualsVisitor equalizer; @@ -46,8 +42,10 @@ public class LinearDictionaryEncoder encoded vector type. * @param decoded vector type, which is also the dictionary type. */ public class SearchDictionaryEncoder implements DictionaryEncoder { - /** - * The dictionary for encoding/decoding. - * It must be sorted. - */ + /** The dictionary for encoding/decoding. It must be sorted. */ private final D dictionary; - /** - * The criteria by which the dictionary is sorted. - */ + /** The criteria by which the dictionary is sorted. */ private final VectorValueComparator comparator; - /** - * A flag indicating if null should be encoded. - */ + /** A flag indicating if null should be encoded. */ private final boolean encodeNull; /** * Constructs a dictionary encoder. + * * @param dictionary the dictionary. It must be in sorted order. * @param comparator the criteria for sorting. */ @@ -57,28 +51,29 @@ public SearchDictionaryEncoder(D dictionary, VectorValueComparator comparator /** * Constructs a dictionary encoder. + * * @param dictionary the dictionary. It must be in sorted order. * @param comparator the criteria for sorting. - * @param encodeNull a flag indicating if null should be encoded. - * It determines the behaviors for processing null values in the input during encoding. - * When a null is encountered in the input, - * 1) If the flag is set to true, the encoder searches for the value in the dictionary, - * and outputs the index in the dictionary. - * 2) If the flag is set to false, the encoder simply produces a null in the output. + * @param encodeNull a flag indicating if null should be encoded. It determines the behaviors for + * processing null values in the input during encoding. When a null is encountered in the + * input, 1) If the flag is set to true, the encoder searches for the value in the dictionary, + * and outputs the index in the dictionary. 2) If the flag is set to false, the encoder simply + * produces a null in the output. */ - public SearchDictionaryEncoder(D dictionary, VectorValueComparator comparator, boolean encodeNull) { + public SearchDictionaryEncoder( + D dictionary, VectorValueComparator comparator, boolean encodeNull) { this.dictionary = dictionary; this.comparator = comparator; this.encodeNull = encodeNull; } /** - * Encodes an input vector by binary search. - * So the algorithm takes O(n * log(m)) time, where n is the length of the input vector, - * and m is the length of the dictionary. + * Encodes an input vector by binary search. So the algorithm takes O(n * log(m)) time, where n is + * the length of the input vector, and m is the length of the dictionary. + * * @param input the input vector. - * @param output the output vector. Note that it must be in a fresh state. At least, - * all its validity bits should be clear. + * @param output the output vector. Note that it must be in a fresh state. At least, all its + * validity bits should be clear. */ @Override public void encode(D input, E output) { diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/dictionary/SearchTreeBasedDictionaryBuilder.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/dictionary/SearchTreeBasedDictionaryBuilder.java index f9cd77daa2e76..fca7df067dcff 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/dictionary/SearchTreeBasedDictionaryBuilder.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/dictionary/SearchTreeBasedDictionaryBuilder.java @@ -14,45 +14,36 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.dictionary; import java.util.TreeSet; - import org.apache.arrow.algorithm.sort.VectorValueComparator; import org.apache.arrow.vector.ValueVector; /** - * This class builds the dictionary based on a binary search tree. - * Each add operation can be finished in O(log(n)) time, - * where n is the current dictionary size. + * This class builds the dictionary based on a binary search tree. Each add operation can be + * finished in O(log(n)) time, where n is the current dictionary size. * * @param the dictionary vector type. */ -public class SearchTreeBasedDictionaryBuilder implements DictionaryBuilder { +public class SearchTreeBasedDictionaryBuilder + implements DictionaryBuilder { - /** - * The dictionary to be built. - */ + /** The dictionary to be built. */ private final V dictionary; - /** - * The criteria for sorting in the search tree. - */ + /** The criteria for sorting in the search tree. */ protected final VectorValueComparator comparator; - /** - * If null should be encoded. - */ + /** If null should be encoded. */ private final boolean encodeNull; - /** - * The search tree for storing the value index. - */ + /** The search tree for storing the value index. */ private TreeSet searchTree; /** * Construct a search tree-based dictionary builder. + * * @param dictionary the dictionary vector. * @param comparator the criteria for value equality. */ @@ -62,11 +53,13 @@ public SearchTreeBasedDictionaryBuilder(V dictionary, VectorValueComparator c /** * Construct a search tree-based dictionary builder. + * * @param dictionary the dictionary vector. * @param comparator the criteria for value equality. * @param encodeNull if null values should be added to the dictionary. */ - public SearchTreeBasedDictionaryBuilder(V dictionary, VectorValueComparator comparator, boolean encodeNull) { + public SearchTreeBasedDictionaryBuilder( + V dictionary, VectorValueComparator comparator, boolean encodeNull) { this.dictionary = dictionary; this.comparator = comparator; this.encodeNull = encodeNull; @@ -76,11 +69,10 @@ public SearchTreeBasedDictionaryBuilder(V dictionary, VectorValueComparator c } /** - * Gets the dictionary built. - * Please note that the dictionary is not in sorted order. - * Instead, its order is determined by the order of element insertion. - * To get the dictionary in sorted order, please use - * {@link SearchTreeBasedDictionaryBuilder#populateSortedDictionary(ValueVector)}. + * Gets the dictionary built. Please note that the dictionary is not in sorted order. Instead, its + * order is determined by the order of element insertion. To get the dictionary in sorted order, + * please use {@link SearchTreeBasedDictionaryBuilder#populateSortedDictionary(ValueVector)}. + * * @return the dictionary. */ @Override @@ -90,6 +82,7 @@ public V getDictionary() { /** * Try to add all values from the target vector to the dictionary. + * * @param targetVector the target vector containing values to probe. * @return the number of values actually added to the dictionary. */ @@ -107,6 +100,7 @@ public int addValues(V targetVector) { /** * Try to add an element from the target vector to the dictionary. + * * @param targetVector the target vector containing new element. * @param targetIndex the index of the new element in the target vector. * @return the index of the new element in the dictionary. @@ -132,8 +126,8 @@ public int addValue(V targetVector, int targetIndex) { } /** - * Gets the sorted dictionary. - * Note that given the binary search tree, the sort can finish in O(n). + * Gets the sorted dictionary. Note that given the binary search tree, the sort can finish in + * O(n). */ public void populateSortedDictionary(V sortedDictionary) { int idx = 0; diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/misc/PartialSumUtils.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/misc/PartialSumUtils.java index f5e95cf1033f5..5492676af1a2e 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/misc/PartialSumUtils.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/misc/PartialSumUtils.java @@ -14,26 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.misc; import org.apache.arrow.vector.BaseIntVector; -/** - * Partial sum related utilities. - */ +/** Partial sum related utilities. */ public class PartialSumUtils { /** - * Converts an input vector to a partial sum vector. - * This is an inverse operation of {@link PartialSumUtils#toDeltaVector(BaseIntVector, BaseIntVector)}. - * Suppose we have input vector a and output vector b. - * Then we have b(0) = sumBase; b(i + 1) = b(i) + a(i) (i = 0, 1, 2, ...). + * Converts an input vector to a partial sum vector. This is an inverse operation of {@link + * PartialSumUtils#toDeltaVector(BaseIntVector, BaseIntVector)}. Suppose we have input vector a + * and output vector b. Then we have b(0) = sumBase; b(i + 1) = b(i) + a(i) (i = 0, 1, 2, ...). + * * @param deltaVector the input vector. * @param partialSumVector the output vector. * @param sumBase the base of the partial sums. */ - public static void toPartialSumVector(BaseIntVector deltaVector, BaseIntVector partialSumVector, long sumBase) { + public static void toPartialSumVector( + BaseIntVector deltaVector, BaseIntVector partialSumVector, long sumBase) { long sum = sumBase; partialSumVector.setWithPossibleTruncate(0, sumBase); @@ -45,10 +43,10 @@ public static void toPartialSumVector(BaseIntVector deltaVector, BaseIntVector p } /** - * Converts an input vector to the delta vector. - * This is an inverse operation of {@link PartialSumUtils#toPartialSumVector(BaseIntVector, BaseIntVector, long)}. - * Suppose we have input vector a and output vector b. - * Then we have b(i) = a(i + 1) - a(i) (i = 0, 1, 2, ...). + * Converts an input vector to the delta vector. This is an inverse operation of {@link + * PartialSumUtils#toPartialSumVector(BaseIntVector, BaseIntVector, long)}. Suppose we have input + * vector a and output vector b. Then we have b(i) = a(i + 1) - a(i) (i = 0, 1, 2, ...). + * * @param partialSumVector the input vector. * @param deltaVector the output vector. */ @@ -61,18 +59,18 @@ public static void toDeltaVector(BaseIntVector partialSumVector, BaseIntVector d } /** - * Given a value and a partial sum vector, finds its position in the partial sum vector. - * In particular, given an integer value a and partial sum vector v, we try to find a - * position i, so that v(i) <= a < v(i + 1). - * The algorithm is based on binary search, so it takes O(log(n)) time, where n is - * the length of the partial sum vector. + * Given a value and a partial sum vector, finds its position in the partial sum vector. In + * particular, given an integer value a and partial sum vector v, we try to find a position i, so + * that v(i) <= a < v(i + 1). The algorithm is based on binary search, so it takes O(log(n)) time, + * where n is the length of the partial sum vector. + * * @param partialSumVector the input partial sum vector. * @param value the value to search. * @return the position in the partial sum vector, if any, or -1, if none is found. */ public static int findPositionInPartialSumVector(BaseIntVector partialSumVector, long value) { - if (value < partialSumVector.getValueAsLong(0) || - value >= partialSumVector.getValueAsLong(partialSumVector.getValueCount() - 1)) { + if (value < partialSumVector.getValueAsLong(0) + || value >= partialSumVector.getValueAsLong(partialSumVector.getValueCount() - 1)) { return -1; } @@ -114,6 +112,5 @@ public static int findPositionInPartialSumVector(BaseIntVector partialSumVector, throw new IllegalStateException("Should never get here"); } - private PartialSumUtils() { - } + private PartialSumUtils() {} } diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/rank/VectorRank.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/rank/VectorRank.java index 43c9a5b010e8c..baa2058ffc51f 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/rank/VectorRank.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/rank/VectorRank.java @@ -14,11 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.rank; import java.util.stream.IntStream; - import org.apache.arrow.algorithm.sort.IndexSorter; import org.apache.arrow.algorithm.sort.VectorValueComparator; import org.apache.arrow.memory.BufferAllocator; @@ -28,21 +26,21 @@ /** * Utility for calculating ranks of vector elements. + * * @param the vector type */ public class VectorRank { private VectorValueComparator comparator; - /** - * Vector indices. - */ + /** Vector indices. */ private IntVector indices; private final BufferAllocator allocator; /** * Constructs a vector rank utility. + * * @param allocator the allocator to use. */ public VectorRank(BufferAllocator allocator) { @@ -50,9 +48,10 @@ public VectorRank(BufferAllocator allocator) { } /** - * Given a rank r, gets the index of the element that is the rth smallest in the vector. - * The operation is performed without changing the vector, and takes O(n) time, - * where n is the length of the vector. + * Given a rank r, gets the index of the element that is the rth smallest in the vector. The + * operation is performed without changing the vector, and takes O(n) time, where n is the length + * of the vector. + * * @param vector the vector from which to get the element index. * @param comparator the criteria for vector element comparison. * @param rank the rank to determine. diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/search/ParallelSearcher.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/search/ParallelSearcher.java index 6226921b22ed6..6a48019edc3eb 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/search/ParallelSearcher.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/search/ParallelSearcher.java @@ -14,49 +14,40 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.search; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; - import org.apache.arrow.algorithm.sort.VectorValueComparator; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.compare.Range; import org.apache.arrow.vector.compare.RangeEqualsVisitor; /** - * Search for a value in the vector by multiple threads. - * This is often used in scenarios where the vector is large or - * low response time is required. + * Search for a value in the vector by multiple threads. This is often used in scenarios where the + * vector is large or low response time is required. + * * @param the vector type. */ public class ParallelSearcher { - /** - * The target vector to search. - */ + /** The target vector to search. */ private final V vector; - /** - * The thread pool. - */ + /** The thread pool. */ private final ExecutorService threadPool; - /** - * The number of threads to use. - */ + /** The number of threads to use. */ private final int numThreads; - /** - * The position of the key in the target vector, if any. - */ + /** The position of the key in the target vector, if any. */ private volatile int keyPosition = -1; /** * Constructs a parallel searcher. + * * @param vector the vector to search. * @param threadPool the thread pool to use. * @param numThreads the number of threads to use. @@ -77,17 +68,17 @@ private CompletableFuture[] initSearch() { } /** - * Search for the key in the target vector. The element-wise comparison is based on - * {@link RangeEqualsVisitor}, so there are two possible results for each element-wise - * comparison: equal and un-equal. + * Search for the key in the target vector. The element-wise comparison is based on {@link + * RangeEqualsVisitor}, so there are two possible results for each element-wise comparison: equal + * and un-equal. + * * @param keyVector the vector containing the search key. * @param keyIndex the index of the search key in the key vector. - * @return the position of a matched value in the target vector, - * or -1 if none is found. Please note that if there are multiple - * matches of the key in the target vector, this method makes no - * guarantees about which instance is returned. - * For an alternative search implementation that always finds the first match of the key, - * see {@link VectorSearcher#linearSearch(ValueVector, VectorValueComparator, ValueVector, int)}. + * @return the position of a matched value in the target vector, or -1 if none is found. Please + * note that if there are multiple matches of the key in the target vector, this method makes + * no guarantees about which instance is returned. For an alternative search implementation + * that always finds the first match of the key, see {@link + * VectorSearcher#linearSearch(ValueVector, VectorValueComparator, ValueVector, int)}. * @throws ExecutionException if an exception occurs in a thread. * @throws InterruptedException if a thread is interrupted. */ @@ -96,36 +87,38 @@ public int search(V keyVector, int keyIndex) throws ExecutionException, Interrup final int valueCount = vector.getValueCount(); for (int i = 0; i < numThreads; i++) { final int tid = i; - Future unused = threadPool.submit(() -> { - // convert to long to avoid overflow - int start = (int) (((long) valueCount) * tid / numThreads); - int end = (int) ((long) valueCount) * (tid + 1) / numThreads; - - if (start >= end) { - // no data assigned to this task. - futures[tid].complete(false); - return; - } - - RangeEqualsVisitor visitor = new RangeEqualsVisitor(vector, keyVector, null); - Range range = new Range(0, 0, 1); - for (int pos = start; pos < end; pos++) { - if (keyPosition != -1) { - // the key has been found by another task - futures[tid].complete(false); - return; - } - range.setLeftStart(pos).setRightStart(keyIndex); - if (visitor.rangeEquals(range)) { - keyPosition = pos; - futures[tid].complete(true); - return; - } - } - - // no match value is found. - futures[tid].complete(false); - }); + Future unused = + threadPool.submit( + () -> { + // convert to long to avoid overflow + int start = (int) (((long) valueCount) * tid / numThreads); + int end = (int) ((long) valueCount) * (tid + 1) / numThreads; + + if (start >= end) { + // no data assigned to this task. + futures[tid].complete(false); + return; + } + + RangeEqualsVisitor visitor = new RangeEqualsVisitor(vector, keyVector, null); + Range range = new Range(0, 0, 1); + for (int pos = start; pos < end; pos++) { + if (keyPosition != -1) { + // the key has been found by another task + futures[tid].complete(false); + return; + } + range.setLeftStart(pos).setRightStart(keyIndex); + if (visitor.rangeEquals(range)) { + keyPosition = pos; + futures[tid].complete(true); + return; + } + } + + // no match value is found. + futures[tid].complete(false); + }); } CompletableFuture.allOf(futures).get(); @@ -133,56 +126,58 @@ public int search(V keyVector, int keyIndex) throws ExecutionException, Interrup } /** - * Search for the key in the target vector. The element-wise comparison is based on - * {@link VectorValueComparator}, so there are three possible results for each element-wise - * comparison: less than, equal to and greater than. + * Search for the key in the target vector. The element-wise comparison is based on {@link + * VectorValueComparator}, so there are three possible results for each element-wise comparison: + * less than, equal to and greater than. + * * @param keyVector the vector containing the search key. * @param keyIndex the index of the search key in the key vector. * @param comparator the comparator for comparing the key against vector elements. - * @return the position of a matched value in the target vector, - * or -1 if none is found. Please note that if there are multiple - * matches of the key in the target vector, this method makes no - * guarantees about which instance is returned. - * For an alternative search implementation that always finds the first match of the key, - * see {@link VectorSearcher#linearSearch(ValueVector, VectorValueComparator, ValueVector, int)}. + * @return the position of a matched value in the target vector, or -1 if none is found. Please + * note that if there are multiple matches of the key in the target vector, this method makes + * no guarantees about which instance is returned. For an alternative search implementation + * that always finds the first match of the key, see {@link + * VectorSearcher#linearSearch(ValueVector, VectorValueComparator, ValueVector, int)}. * @throws ExecutionException if an exception occurs in a thread. * @throws InterruptedException if a thread is interrupted. */ - public int search( - V keyVector, int keyIndex, VectorValueComparator comparator) throws ExecutionException, InterruptedException { + public int search(V keyVector, int keyIndex, VectorValueComparator comparator) + throws ExecutionException, InterruptedException { final CompletableFuture[] futures = initSearch(); final int valueCount = vector.getValueCount(); for (int i = 0; i < numThreads; i++) { final int tid = i; - Future unused = threadPool.submit(() -> { - // convert to long to avoid overflow - int start = (int) (((long) valueCount) * tid / numThreads); - int end = (int) ((long) valueCount) * (tid + 1) / numThreads; - - if (start >= end) { - // no data assigned to this task. - futures[tid].complete(false); - return; - } - - VectorValueComparator localComparator = comparator.createNew(); - localComparator.attachVectors(vector, keyVector); - for (int pos = start; pos < end; pos++) { - if (keyPosition != -1) { - // the key has been found by another task - futures[tid].complete(false); - return; - } - if (localComparator.compare(pos, keyIndex) == 0) { - keyPosition = pos; - futures[tid].complete(true); - return; - } - } - - // no match value is found. - futures[tid].complete(false); - }); + Future unused = + threadPool.submit( + () -> { + // convert to long to avoid overflow + int start = (int) (((long) valueCount) * tid / numThreads); + int end = (int) ((long) valueCount) * (tid + 1) / numThreads; + + if (start >= end) { + // no data assigned to this task. + futures[tid].complete(false); + return; + } + + VectorValueComparator localComparator = comparator.createNew(); + localComparator.attachVectors(vector, keyVector); + for (int pos = start; pos < end; pos++) { + if (keyPosition != -1) { + // the key has been found by another task + futures[tid].complete(false); + return; + } + if (localComparator.compare(pos, keyIndex) == 0) { + keyPosition = pos; + futures[tid].complete(true); + return; + } + } + + // no match value is found. + futures[tid].complete(false); + }); } CompletableFuture.allOf(futures).get(); diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/search/VectorRangeSearcher.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/search/VectorRangeSearcher.java index 249194843f101..c7905dd8956c8 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/search/VectorRangeSearcher.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/search/VectorRangeSearcher.java @@ -1,108 +1,105 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.arrow.algorithm.search; - -import org.apache.arrow.algorithm.sort.VectorValueComparator; -import org.apache.arrow.vector.ValueVector; - -/** - * Search for the range of a particular element in the target vector. - */ -public class VectorRangeSearcher { - - /** - * Result returned when a search fails. - */ - public static final int SEARCH_FAIL_RESULT = -1; - - /** - * Search for the first occurrence of an element. - * The search is based on the binary search algorithm. So the target vector must be sorted. - * @param targetVector the vector from which to perform the search. - * @param comparator the criterion for the comparison. - * @param keyVector the vector containing the element to search. - * @param keyIndex the index of the search key in the key vector. - * @param the vector type. - * @return the index of the first matched element if any, and -1 otherwise. - */ - public static int getFirstMatch( - V targetVector, VectorValueComparator comparator, V keyVector, int keyIndex) { - comparator.attachVectors(keyVector, targetVector); - - int ret = SEARCH_FAIL_RESULT; - - int low = 0; - int high = targetVector.getValueCount() - 1; - - while (low <= high) { - int mid = low + (high - low) / 2; - int result = comparator.compare(keyIndex, mid); - if (result < 0) { - // the key is smaller - high = mid - 1; - } else if (result > 0) { - // the key is larger - low = mid + 1; - } else { - // an equal element is found - // continue to go left-ward - ret = mid; - high = mid - 1; - } - } - return ret; - } - - /** - * Search for the last occurrence of an element. - * The search is based on the binary search algorithm. So the target vector must be sorted. - * @param targetVector the vector from which to perform the search. - * @param comparator the criterion for the comparison. - * @param keyVector the vector containing the element to search. - * @param keyIndex the index of the search key in the key vector. - * @param the vector type. - * @return the index of the last matched element if any, and -1 otherwise. - */ - public static int getLastMatch( - V targetVector, VectorValueComparator comparator, V keyVector, int keyIndex) { - comparator.attachVectors(keyVector, targetVector); - - int ret = SEARCH_FAIL_RESULT; - - int low = 0; - int high = targetVector.getValueCount() - 1; - - while (low <= high) { - int mid = low + (high - low) / 2; - int result = comparator.compare(keyIndex, mid); - if (result < 0) { - // the key is smaller - high = mid - 1; - } else if (result > 0) { - // the key is larger - low = mid + 1; - } else { - // an equal element is found, - // continue to go right-ward - ret = mid; - low = mid + 1; - } - } - return ret; - } -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.algorithm.search; + +import org.apache.arrow.algorithm.sort.VectorValueComparator; +import org.apache.arrow.vector.ValueVector; + +/** Search for the range of a particular element in the target vector. */ +public class VectorRangeSearcher { + + /** Result returned when a search fails. */ + public static final int SEARCH_FAIL_RESULT = -1; + + /** + * Search for the first occurrence of an element. The search is based on the binary search + * algorithm. So the target vector must be sorted. + * + * @param targetVector the vector from which to perform the search. + * @param comparator the criterion for the comparison. + * @param keyVector the vector containing the element to search. + * @param keyIndex the index of the search key in the key vector. + * @param the vector type. + * @return the index of the first matched element if any, and -1 otherwise. + */ + public static int getFirstMatch( + V targetVector, VectorValueComparator comparator, V keyVector, int keyIndex) { + comparator.attachVectors(keyVector, targetVector); + + int ret = SEARCH_FAIL_RESULT; + + int low = 0; + int high = targetVector.getValueCount() - 1; + + while (low <= high) { + int mid = low + (high - low) / 2; + int result = comparator.compare(keyIndex, mid); + if (result < 0) { + // the key is smaller + high = mid - 1; + } else if (result > 0) { + // the key is larger + low = mid + 1; + } else { + // an equal element is found + // continue to go left-ward + ret = mid; + high = mid - 1; + } + } + return ret; + } + + /** + * Search for the last occurrence of an element. The search is based on the binary search + * algorithm. So the target vector must be sorted. + * + * @param targetVector the vector from which to perform the search. + * @param comparator the criterion for the comparison. + * @param keyVector the vector containing the element to search. + * @param keyIndex the index of the search key in the key vector. + * @param the vector type. + * @return the index of the last matched element if any, and -1 otherwise. + */ + public static int getLastMatch( + V targetVector, VectorValueComparator comparator, V keyVector, int keyIndex) { + comparator.attachVectors(keyVector, targetVector); + + int ret = SEARCH_FAIL_RESULT; + + int low = 0; + int high = targetVector.getValueCount() - 1; + + while (low <= high) { + int mid = low + (high - low) / 2; + int result = comparator.compare(keyIndex, mid); + if (result < 0) { + // the key is smaller + high = mid - 1; + } else if (result > 0) { + // the key is larger + low = mid + 1; + } else { + // an equal element is found, + // continue to go right-ward + ret = mid; + low = mid + 1; + } + } + return ret; + } +} diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/search/VectorSearcher.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/search/VectorSearcher.java index 646bca01bb81d..dd0b4de5d8677 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/search/VectorSearcher.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/search/VectorSearcher.java @@ -14,25 +14,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.search; import org.apache.arrow.algorithm.sort.VectorValueComparator; import org.apache.arrow.vector.ValueVector; -/** - * Search for a particular element in the vector. - */ +/** Search for a particular element in the vector. */ public final class VectorSearcher { - /** - * Result returned when a search fails. - */ + /** Result returned when a search fails. */ public static final int SEARCH_FAIL_RESULT = -1; /** - * Search for a particular element from the key vector in the target vector by binary search. - * The target vector must be sorted. + * Search for a particular element from the key vector in the target vector by binary search. The + * target vector must be sorted. + * * @param targetVector the vector from which to perform the sort. * @param comparator the criterion for the sort. * @param keyVector the vector containing the element to search. @@ -41,7 +37,7 @@ public final class VectorSearcher { * @return the index of a matched element if any, and -1 otherwise. */ public static int binarySearch( - V targetVector, VectorValueComparator comparator, V keyVector, int keyIndex) { + V targetVector, VectorValueComparator comparator, V keyVector, int keyIndex) { comparator.attachVectors(keyVector, targetVector); // perform binary search @@ -63,7 +59,9 @@ public static int binarySearch( } /** - * Search for a particular element from the key vector in the target vector by traversing the vector in sequence. + * Search for a particular element from the key vector in the target vector by traversing the + * vector in sequence. + * * @param targetVector the vector from which to perform the search. * @param comparator the criterion for element equality. * @param keyVector the vector containing the element to search. @@ -72,7 +70,7 @@ public static int binarySearch( * @return the index of a matched element if any, and -1 otherwise. */ public static int linearSearch( - V targetVector, VectorValueComparator comparator, V keyVector, int keyIndex) { + V targetVector, VectorValueComparator comparator, V keyVector, int keyIndex) { comparator.attachVectors(keyVector, targetVector); for (int i = 0; i < targetVector.getValueCount(); i++) { if (comparator.compare(keyIndex, i) == 0) { @@ -82,7 +80,5 @@ public static int linearSearch( return SEARCH_FAIL_RESULT; } - private VectorSearcher() { - - } + private VectorSearcher() {} } diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/CompositeVectorComparator.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/CompositeVectorComparator.java index ec74598e0eebf..77093d87bc489 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/CompositeVectorComparator.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/CompositeVectorComparator.java @@ -14,20 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.sort; import org.apache.arrow.vector.ValueVector; /** - * A composite vector comparator compares a number of vectors - * by a number of inner comparators. - *

    - * It works by first using the first comparator, if a non-zero value - * is returned, it simply returns it. Otherwise, it uses the second comparator, - * and so on, until a non-zero value is produced, or all inner comparators have - * been used. - *

    + * A composite vector comparator compares a number of vectors by a number of inner comparators. + * + *

    It works by first using the first comparator, if a non-zero value is returned, it simply + * returns it. Otherwise, it uses the second comparator, and so on, until a non-zero value is + * produced, or all inner comparators have been used. */ public class CompositeVectorComparator extends VectorValueComparator { @@ -62,7 +58,8 @@ public int compare(int index1, int index2) { @Override public VectorValueComparator createNew() { - VectorValueComparator[] newInnerComparators = new VectorValueComparator[innerComparators.length]; + VectorValueComparator[] newInnerComparators = + new VectorValueComparator[innerComparators.length]; for (int i = 0; i < innerComparators.length; i++) { newInnerComparators[i] = innerComparators[i].createNew(); } diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/DefaultVectorComparators.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/DefaultVectorComparators.java index 588876aa99059..ec650cd9dc88b 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/DefaultVectorComparators.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/DefaultVectorComparators.java @@ -14,14 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.sort; import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.OFFSET_WIDTH; import java.math.BigDecimal; import java.time.Duration; - import org.apache.arrow.memory.util.ArrowBufPointer; import org.apache.arrow.memory.util.ByteFunctionHelpers; import org.apache.arrow.vector.BaseFixedWidthVector; @@ -56,13 +54,12 @@ import org.apache.arrow.vector.complex.RepeatedValueVector; import org.apache.arrow.vector.holders.NullableFixedSizeBinaryHolder; -/** - * Default comparator implementations for different types of vectors. - */ +/** Default comparator implementations for different types of vectors. */ public class DefaultVectorComparators { /** * Create the default comparator for the vector. + * * @param vector the vector. * @param the vector type. * @return the default comparator. @@ -104,7 +101,8 @@ public static VectorValueComparator createDefaultComp } else if (vector instanceof IntervalDayVector) { return (VectorValueComparator) new IntervalDayComparator(); } else if (vector instanceof IntervalMonthDayNanoVector) { - throw new IllegalArgumentException("No default comparator for " + vector.getClass().getCanonicalName()); + throw new IllegalArgumentException( + "No default comparator for " + vector.getClass().getCanonicalName()); } else if (vector instanceof TimeMicroVector) { return (VectorValueComparator) new TimeMicroComparator(); } else if (vector instanceof TimeMilliVector) { @@ -122,7 +120,7 @@ public static VectorValueComparator createDefaultComp return (VectorValueComparator) new VariableWidthComparator(); } else if (vector instanceof RepeatedValueVector) { VectorValueComparator innerComparator = - createDefaultComparator(((RepeatedValueVector) vector).getDataVector()); + createDefaultComparator(((RepeatedValueVector) vector).getDataVector()); return new RepeatedValueComparator(innerComparator); } else if (vector instanceof FixedSizeListVector) { VectorValueComparator innerComparator = @@ -132,13 +130,11 @@ public static VectorValueComparator createDefaultComp return (VectorValueComparator) new NullComparator(); } - throw new IllegalArgumentException("No default comparator for " + vector.getClass().getCanonicalName()); + throw new IllegalArgumentException( + "No default comparator for " + vector.getClass().getCanonicalName()); } - /** - * Default comparator for bytes. - * The comparison is based on values, with null comes first. - */ + /** Default comparator for bytes. The comparison is based on values, with null comes first. */ public static class ByteComparator extends VectorValueComparator { public ByteComparator() { @@ -159,8 +155,8 @@ public VectorValueComparator createNew() { } /** - * Default comparator for short integers. - * The comparison is based on values, with null comes first. + * Default comparator for short integers. The comparison is based on values, with null comes + * first. */ public static class ShortComparator extends VectorValueComparator { @@ -182,8 +178,8 @@ public VectorValueComparator createNew() { } /** - * Default comparator for 32-bit integers. - * The comparison is based on int values, with null comes first. + * Default comparator for 32-bit integers. The comparison is based on int values, with null comes + * first. */ public static class IntComparator extends VectorValueComparator { @@ -205,8 +201,7 @@ public VectorValueComparator createNew() { } /** - * Default comparator for long integers. - * The comparison is based on values, with null comes first. + * Default comparator for long integers. The comparison is based on values, with null comes first. */ public static class LongComparator extends VectorValueComparator { @@ -229,8 +224,8 @@ public VectorValueComparator createNew() { } /** - * Default comparator for unsigned bytes. - * The comparison is based on values, with null comes first. + * Default comparator for unsigned bytes. The comparison is based on values, with null comes + * first. */ public static class UInt1Comparator extends VectorValueComparator { @@ -253,8 +248,8 @@ public VectorValueComparator createNew() { } /** - * Default comparator for unsigned short integer. - * The comparison is based on values, with null comes first. + * Default comparator for unsigned short integer. The comparison is based on values, with null + * comes first. */ public static class UInt2Comparator extends VectorValueComparator { @@ -280,8 +275,8 @@ public VectorValueComparator createNew() { } /** - * Default comparator for unsigned integer. - * The comparison is based on values, with null comes first. + * Default comparator for unsigned integer. The comparison is based on values, with null comes + * first. */ public static class UInt4Comparator extends VectorValueComparator { @@ -303,8 +298,8 @@ public VectorValueComparator createNew() { } /** - * Default comparator for unsigned long integer. - * The comparison is based on values, with null comes first. + * Default comparator for unsigned long integer. The comparison is based on values, with null + * comes first. */ public static class UInt8Comparator extends VectorValueComparator { @@ -326,8 +321,7 @@ public VectorValueComparator createNew() { } /** - * Default comparator for float type. - * The comparison is based on values, with null comes first. + * Default comparator for float type. The comparison is based on values, with null comes first. */ public static class Float4Comparator extends VectorValueComparator { @@ -363,8 +357,7 @@ public VectorValueComparator createNew() { } /** - * Default comparator for double type. - * The comparison is based on values, with null comes first. + * Default comparator for double type. The comparison is based on values, with null comes first. */ public static class Float8Comparator extends VectorValueComparator { @@ -399,10 +392,7 @@ public VectorValueComparator createNew() { } } - /** - * Default comparator for bit type. - * The comparison is based on values, with null comes first. - */ + /** Default comparator for bit type. The comparison is based on values, with null comes first. */ public static class BitComparator extends VectorValueComparator { public BitComparator() { @@ -424,8 +414,7 @@ public VectorValueComparator createNew() { } /** - * Default comparator for DateDay type. - * The comparison is based on values, with null comes first. + * Default comparator for DateDay type. The comparison is based on values, with null comes first. */ public static class DateDayComparator extends VectorValueComparator { @@ -447,8 +436,8 @@ public VectorValueComparator createNew() { } /** - * Default comparator for DateMilli type. - * The comparison is based on values, with null comes first. + * Default comparator for DateMilli type. The comparison is based on values, with null comes + * first. */ public static class DateMilliComparator extends VectorValueComparator { @@ -471,8 +460,8 @@ public VectorValueComparator createNew() { } /** - * Default comparator for Decimal256 type. - * The comparison is based on values, with null comes first. + * Default comparator for Decimal256 type. The comparison is based on values, with null comes + * first. */ public static class Decimal256Comparator extends VectorValueComparator { @@ -495,8 +484,7 @@ public VectorValueComparator createNew() { } /** - * Default comparator for Decimal type. - * The comparison is based on values, with null comes first. + * Default comparator for Decimal type. The comparison is based on values, with null comes first. */ public static class DecimalComparator extends VectorValueComparator { @@ -519,8 +507,7 @@ public VectorValueComparator createNew() { } /** - * Default comparator for Duration type. - * The comparison is based on values, with null comes first. + * Default comparator for Duration type. The comparison is based on values, with null comes first. */ public static class DurationComparator extends VectorValueComparator { @@ -543,8 +530,8 @@ public VectorValueComparator createNew() { } /** - * Default comparator for IntervalDay type. - * The comparison is based on values, with null comes first. + * Default comparator for IntervalDay type. The comparison is based on values, with null comes + * first. */ public static class IntervalDayComparator extends VectorValueComparator { @@ -567,8 +554,8 @@ public VectorValueComparator createNew() { } /** - * Default comparator for TimeMicro type. - * The comparison is based on values, with null comes first. + * Default comparator for TimeMicro type. The comparison is based on values, with null comes + * first. */ public static class TimeMicroComparator extends VectorValueComparator { @@ -591,8 +578,8 @@ public VectorValueComparator createNew() { } /** - * Default comparator for TimeMilli type. - * The comparison is based on values, with null comes first. + * Default comparator for TimeMilli type. The comparison is based on values, with null comes + * first. */ public static class TimeMilliComparator extends VectorValueComparator { @@ -615,8 +602,7 @@ public VectorValueComparator createNew() { } /** - * Default comparator for TimeNano type. - * The comparison is based on values, with null comes first. + * Default comparator for TimeNano type. The comparison is based on values, with null comes first. */ public static class TimeNanoComparator extends VectorValueComparator { @@ -639,8 +625,7 @@ public VectorValueComparator createNew() { } /** - * Default comparator for TimeSec type. - * The comparison is based on values, with null comes first. + * Default comparator for TimeSec type. The comparison is based on values, with null comes first. */ public static class TimeSecComparator extends VectorValueComparator { @@ -663,8 +648,7 @@ public VectorValueComparator createNew() { } /** - * Default comparator for TimeSec type. - * The comparison is based on values, with null comes first. + * Default comparator for TimeSec type. The comparison is based on values, with null comes first. */ public static class TimeStampComparator extends VectorValueComparator { @@ -687,10 +671,11 @@ public VectorValueComparator createNew() { } /** - * Default comparator for {@link org.apache.arrow.vector.FixedSizeBinaryVector}. - * The comparison is in lexicographic order, with null comes first. + * Default comparator for {@link org.apache.arrow.vector.FixedSizeBinaryVector}. The comparison is + * in lexicographic order, with null comes first. */ - public static class FixedSizeBinaryComparator extends VectorValueComparator { + public static class FixedSizeBinaryComparator + extends VectorValueComparator { @Override public int compare(int index1, int index2) { @@ -720,9 +705,7 @@ public VectorValueComparator createNew() { } } - /** - * Default comparator for {@link org.apache.arrow.vector.NullVector}. - */ + /** Default comparator for {@link org.apache.arrow.vector.NullVector}. */ public static class NullComparator extends VectorValueComparator { @Override public int compare(int index1, int index2) { @@ -742,8 +725,8 @@ public VectorValueComparator createNew() { } /** - * Default comparator for {@link org.apache.arrow.vector.VariableWidthVector}. - * The comparison is in lexicographic order, with null comes first. + * Default comparator for {@link org.apache.arrow.vector.VariableWidthVector}. The comparison is + * in lexicographic order, with null comes first. */ public static class VariableWidthComparator extends VectorValueComparator { @@ -772,12 +755,13 @@ public VectorValueComparator createNew() { } /** - * Default comparator for {@link RepeatedValueVector}. - * It works by comparing the underlying vector in a lexicographic order. + * Default comparator for {@link RepeatedValueVector}. It works by comparing the underlying vector + * in a lexicographic order. + * * @param inner vector type. */ public static class RepeatedValueComparator - extends VectorValueComparator { + extends VectorValueComparator { private final VectorValueComparator innerComparator; @@ -823,8 +807,9 @@ public void attachVectors(RepeatedValueVector vector1, RepeatedValueVector vecto } /** - * Default comparator for {@link RepeatedValueVector}. - * It works by comparing the underlying vector in a lexicographic order. + * Default comparator for {@link RepeatedValueVector}. It works by comparing the underlying vector + * in a lexicographic order. + * * @param inner vector type. */ public static class FixedSizeListComparator @@ -869,6 +854,5 @@ public void attachVectors(FixedSizeListVector vector1, FixedSizeListVector vecto } } - private DefaultVectorComparators() { - } + private DefaultVectorComparators() {} } diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/FixedWidthInPlaceVectorSorter.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/FixedWidthInPlaceVectorSorter.java index aaa7ba117c3ba..ea2b344a1eabb 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/FixedWidthInPlaceVectorSorter.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/FixedWidthInPlaceVectorSorter.java @@ -14,20 +14,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.sort; import org.apache.arrow.vector.BaseFixedWidthVector; /** - * Default in-place sorter for fixed-width vectors. - * It is based on quick-sort, with average time complexity O(n*log(n)). + * Default in-place sorter for fixed-width vectors. It is based on quick-sort, with average time + * complexity O(n*log(n)). + * * @param vector type. */ -public class FixedWidthInPlaceVectorSorter implements InPlaceVectorSorter { +public class FixedWidthInPlaceVectorSorter + implements InPlaceVectorSorter { /** - * If the number of items is smaller than this threshold, we will use another algorithm to sort the data. + * If the number of items is smaller than this threshold, we will use another algorithm to sort + * the data. */ public static final int CHANGE_ALGORITHM_THRESHOLD = 15; @@ -35,15 +37,10 @@ public class FixedWidthInPlaceVectorSorter imple VectorValueComparator comparator; - /** - * The vector to sort. - */ + /** The vector to sort. */ V vec; - /** - * The buffer to hold the pivot. - * It always has length 1. - */ + /** The buffer to hold the pivot. It always has length 1. */ V pivotBuffer; @Override @@ -99,9 +96,7 @@ private void quickSort() { } } - /** - * Select the pivot as the median of 3 samples. - */ + /** Select the pivot as the median of 3 samples. */ void choosePivot(int low, int high) { // we need at least 3 items if (high - low + 1 < STOP_CHOOSING_PIVOT_THRESHOLD) { diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/FixedWidthOutOfPlaceVectorSorter.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/FixedWidthOutOfPlaceVectorSorter.java index 05a4585792dc2..817e890a5abe1 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/FixedWidthOutOfPlaceVectorSorter.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/FixedWidthOutOfPlaceVectorSorter.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.sort; import org.apache.arrow.memory.ArrowBuf; @@ -26,18 +25,21 @@ import org.apache.arrow.vector.IntVector; /** - * Default out-of-place sorter for fixed-width vectors. - * It is an out-of-place sort, with time complexity O(n*log(n)). + * Default out-of-place sorter for fixed-width vectors. It is an out-of-place sort, with time + * complexity O(n*log(n)). + * * @param vector type. */ -public class FixedWidthOutOfPlaceVectorSorter implements OutOfPlaceVectorSorter { +public class FixedWidthOutOfPlaceVectorSorter + implements OutOfPlaceVectorSorter { protected IndexSorter indexSorter = new IndexSorter<>(); @Override public void sortOutOfPlace(V srcVector, V dstVector, VectorValueComparator comparator) { if (srcVector instanceof BitVector) { - throw new IllegalArgumentException("BitVector is not supported with FixedWidthOutOfPlaceVectorSorter."); + throw new IllegalArgumentException( + "BitVector is not supported with FixedWidthOutOfPlaceVectorSorter."); } comparator.attachVector(srcVector); @@ -49,15 +51,18 @@ public void sortOutOfPlace(V srcVector, V dstVector, VectorValueComparator co ArrowBuf dstValueBuffer = dstVector.getDataBuffer(); // check buffer size - Preconditions.checkArgument(dstValidityBuffer.capacity() * 8 >= srcVector.getValueCount(), - "Not enough capacity for the validity buffer of the dst vector. " + - "Expected capacity %s, actual capacity %s", - (srcVector.getValueCount() + 7) / 8, dstValidityBuffer.capacity()); + Preconditions.checkArgument( + dstValidityBuffer.capacity() * 8 >= srcVector.getValueCount(), + "Not enough capacity for the validity buffer of the dst vector. " + + "Expected capacity %s, actual capacity %s", + (srcVector.getValueCount() + 7) / 8, + dstValidityBuffer.capacity()); Preconditions.checkArgument( dstValueBuffer.capacity() >= srcVector.getValueCount() * ((long) srcVector.getTypeWidth()), - "Not enough capacity for the data buffer of the dst vector. " + - "Expected capacity %s, actual capacity %s", - srcVector.getValueCount() * srcVector.getTypeWidth(), dstValueBuffer.capacity()); + "Not enough capacity for the data buffer of the dst vector. " + + "Expected capacity %s, actual capacity %s", + srcVector.getValueCount() * srcVector.getTypeWidth(), + dstValueBuffer.capacity()); // sort value indices try (IntVector sortedIndices = new IntVector("", srcVector.getAllocator())) { @@ -73,9 +78,9 @@ public void sortOutOfPlace(V srcVector, V dstVector, VectorValueComparator co } else { BitVectorHelper.setBit(dstValidityBuffer, dstIndex); MemoryUtil.UNSAFE.copyMemory( - srcValueBuffer.memoryAddress() + srcIndex * ((long) valueWidth), - dstValueBuffer.memoryAddress() + dstIndex * ((long) valueWidth), - valueWidth); + srcValueBuffer.memoryAddress() + srcIndex * ((long) valueWidth), + dstValueBuffer.memoryAddress() + dstIndex * ((long) valueWidth), + valueWidth); } } } diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/GeneralOutOfPlaceVectorSorter.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/GeneralOutOfPlaceVectorSorter.java index 9ea39f638aebe..18f5e94314f83 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/GeneralOutOfPlaceVectorSorter.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/GeneralOutOfPlaceVectorSorter.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.sort; import org.apache.arrow.util.Preconditions; @@ -22,23 +21,26 @@ import org.apache.arrow.vector.ValueVector; /** - * An out-of-place sorter for vectors of arbitrary type, with time complexity O(n*log(n)). - * Since it does not make any assumptions about the memory layout of the vector, its performance - * can be sub-optimal. So if another sorter is applicable ({@link FixedWidthInPlaceVectorSorter}), - * it should be used in preference. + * An out-of-place sorter for vectors of arbitrary type, with time complexity O(n*log(n)). Since it + * does not make any assumptions about the memory layout of the vector, its performance can be + * sub-optimal. So if another sorter is applicable ({@link FixedWidthInPlaceVectorSorter}), it + * should be used in preference. * * @param vector type. */ -public class GeneralOutOfPlaceVectorSorter implements OutOfPlaceVectorSorter { +public class GeneralOutOfPlaceVectorSorter + implements OutOfPlaceVectorSorter { @Override public void sortOutOfPlace(V srcVector, V dstVector, VectorValueComparator comparator) { comparator.attachVector(srcVector); // check vector capacity - Preconditions.checkArgument(dstVector.getValueCapacity() >= srcVector.getValueCount(), - "Not enough capacity for the target vector. " + - "Expected capacity %s, actual capacity %s", srcVector.getValueCount(), dstVector.getValueCapacity()); + Preconditions.checkArgument( + dstVector.getValueCapacity() >= srcVector.getValueCount(), + "Not enough capacity for the target vector. " + "Expected capacity %s, actual capacity %s", + srcVector.getValueCount(), + dstVector.getValueCapacity()); // sort value indices try (IntVector sortedIndices = new IntVector("", srcVector.getAllocator())) { diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/InPlaceVectorSorter.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/InPlaceVectorSorter.java index 19817fe76b8ec..ba41bb9e4eac7 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/InPlaceVectorSorter.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/InPlaceVectorSorter.java @@ -14,15 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.sort; import org.apache.arrow.vector.ValueVector; /** - * Basic interface for sorting a vector in-place. - * That is, the sorting is performed by modifying the input vector, - * without creating a new sorted vector. + * Basic interface for sorting a vector in-place. That is, the sorting is performed by modifying the + * input vector, without creating a new sorted vector. * * @param the vector type. */ @@ -30,6 +28,7 @@ public interface InPlaceVectorSorter { /** * Sort a vector in-place. + * * @param vec the vector to sort. * @param comparator the criteria for sort. */ diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/IndexSorter.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/IndexSorter.java index 3072717f43123..b8ce3289d2889 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/IndexSorter.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/IndexSorter.java @@ -14,39 +14,35 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.sort; import java.util.stream.IntStream; - import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.ValueVector; /** * Sorter for the indices of a vector. + * * @param vector type. */ public class IndexSorter { /** - * If the number of items is smaller than this threshold, we will use another algorithm to sort the data. + * If the number of items is smaller than this threshold, we will use another algorithm to sort + * the data. */ public static final int CHANGE_ALGORITHM_THRESHOLD = 15; - /** - * Comparator for vector indices. - */ + /** Comparator for vector indices. */ private VectorValueComparator comparator; - /** - * Vector indices to sort. - */ + /** Vector indices to sort. */ private IntVector indices; /** - * Sorts indices, by quick-sort. Suppose the vector is denoted by v. - * After calling this method, the following relations hold: - * v(indices[0]) <= v(indices[1]) <= ... + * Sorts indices, by quick-sort. Suppose the vector is denoted by v. After calling this method, + * the following relations hold: v(indices[0]) <= v(indices[1]) <= ... + * * @param vector the vector whose indices need to be sorted. * @param indices the vector for storing the sorted indices. * @param comparator the comparator to sort indices. @@ -100,11 +96,9 @@ private void quickSort() { } } - /** - * Select the pivot as the median of 3 samples. - */ + /** Select the pivot as the median of 3 samples. */ static int choosePivot( - int low, int high, IntVector indices, VectorValueComparator comparator) { + int low, int high, IntVector indices, VectorValueComparator comparator) { // we need at least 3 items if (high - low + 1 < FixedWidthInPlaceVectorSorter.STOP_CHOOSING_PIVOT_THRESHOLD) { return indices.get(low); @@ -149,8 +143,9 @@ static int choosePivot( /** * Partition a range of values in a vector into two parts, with elements in one part smaller than - * elements from the other part. The partition is based on the element indices, so it does - * not modify the underlying vector. + * elements from the other part. The partition is based on the element indices, so it does not + * modify the underlying vector. + * * @param low the lower bound of the range. * @param high the upper bound of the range. * @param indices vector element indices. @@ -159,7 +154,7 @@ static int choosePivot( * @return the index of the split point. */ public static int partition( - int low, int high, IntVector indices, VectorValueComparator comparator) { + int low, int high, IntVector indices, VectorValueComparator comparator) { int pivotIndex = choosePivot(low, high, indices, comparator); while (low < high) { diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/InsertionSorter.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/InsertionSorter.java index dc12a5fefdb65..c058636d66d1e 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/InsertionSorter.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/InsertionSorter.java @@ -14,27 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.sort; import org.apache.arrow.vector.BaseFixedWidthVector; import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.ValueVector; -/** - * Insertion sorter. - */ +/** Insertion sorter. */ class InsertionSorter { /** * Sorts the range of a vector by insertion sort. * - * @param vector the vector to be sorted. - * @param startIdx the start index of the range (inclusive). - * @param endIdx the end index of the range (inclusive). - * @param buffer an extra buffer with capacity 1 to hold the current key. + * @param vector the vector to be sorted. + * @param startIdx the start index of the range (inclusive). + * @param endIdx the end index of the range (inclusive). + * @param buffer an extra buffer with capacity 1 to hold the current key. * @param comparator the criteria for vector element comparison. - * @param the vector type. + * @param the vector type. */ static void insertionSort( V vector, int startIdx, int endIdx, VectorValueComparator comparator, V buffer) { @@ -53,11 +50,11 @@ static void insertionSort( /** * Sorts the range of vector indices by insertion sort. * - * @param indices the vector indices. - * @param startIdx the start index of the range (inclusive). - * @param endIdx the end index of the range (inclusive). + * @param indices the vector indices. + * @param startIdx the start index of the range (inclusive). + * @param endIdx the end index of the range (inclusive). * @param comparator the criteria for vector element comparison. - * @param the vector type. + * @param the vector type. */ static void insertionSort( IntVector indices, int startIdx, int endIdx, VectorValueComparator comparator) { diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/OffHeapIntStack.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/OffHeapIntStack.java index df96121f1f8f7..ccb7bea4e2bd3 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/OffHeapIntStack.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/OffHeapIntStack.java @@ -14,15 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.sort; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.IntVector; -/** - * An off heap implementation of stack with int elements. - */ +/** An off heap implementation of stack with int elements. */ class OffHeapIntStack implements AutoCloseable { private static final int INIT_SIZE = 128; diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/OutOfPlaceVectorSorter.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/OutOfPlaceVectorSorter.java index 41d6dadc49147..b18e9b35d0895 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/OutOfPlaceVectorSorter.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/OutOfPlaceVectorSorter.java @@ -14,21 +14,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.sort; import org.apache.arrow.vector.ValueVector; /** - * Basic interface for sorting a vector out-of-place. - * That is, the sorting is performed on a newly-created vector, - * and the original vector is not modified. + * Basic interface for sorting a vector out-of-place. That is, the sorting is performed on a + * newly-created vector, and the original vector is not modified. + * * @param the vector type. */ public interface OutOfPlaceVectorSorter { /** * Sort a vector out-of-place. + * * @param inVec the input vector. * @param outVec the output vector, which has the same size as the input vector. * @param comparator the criteria for sort. diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/StableVectorComparator.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/StableVectorComparator.java index 0b0c3bd55b271..3fcfa5f8f215c 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/StableVectorComparator.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/StableVectorComparator.java @@ -14,17 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.sort; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.ValueVector; /** - * Stable sorter. It compares values like ordinary comparators. - * However, when values are equal, it breaks ties by the value indices. - * Therefore, sort algorithms using this comparator always produce + * Stable sorter. It compares values like ordinary comparators. However, when values are equal, it + * breaks ties by the value indices. Therefore, sort algorithms using this comparator always produce * stable sort results. + * * @param type of the vector. */ public class StableVectorComparator extends VectorValueComparator { @@ -33,6 +32,7 @@ public class StableVectorComparator extends VectorValueCo /** * Constructs a stable comparator from a given comparator. + * * @param innerComparator the comparator to convert to stable comparator.. */ public StableVectorComparator(VectorValueComparator innerComparator) { @@ -47,8 +47,9 @@ public void attachVector(V vector) { @Override public void attachVectors(V vector1, V vector2) { - Preconditions.checkArgument(vector1 == vector2, - "Stable comparator only supports comparing values from the same vector"); + Preconditions.checkArgument( + vector1 == vector2, + "Stable comparator only supports comparing values from the same vector"); super.attachVectors(vector1, vector2); innerComparator.attachVectors(vector1, vector2); } diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/VariableWidthOutOfPlaceVectorSorter.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/VariableWidthOutOfPlaceVectorSorter.java index 863b07c348ef2..8f58dc0dcee0f 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/VariableWidthOutOfPlaceVectorSorter.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/VariableWidthOutOfPlaceVectorSorter.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.sort; import org.apache.arrow.memory.ArrowBuf; @@ -25,12 +24,13 @@ import org.apache.arrow.vector.IntVector; /** - * Default sorter for variable-width vectors. - * It is an out-of-place sort, with time complexity O(n*log(n)). + * Default sorter for variable-width vectors. It is an out-of-place sort, with time complexity + * O(n*log(n)). + * * @param vector type. */ public class VariableWidthOutOfPlaceVectorSorter - implements OutOfPlaceVectorSorter { + implements OutOfPlaceVectorSorter { protected IndexSorter indexSorter = new IndexSorter<>(); @@ -46,20 +46,29 @@ public void sortOutOfPlace(V srcVector, V dstVector, VectorValueComparator co ArrowBuf dstOffsetBuffer = dstVector.getOffsetBuffer(); // check buffer size - Preconditions.checkArgument(dstValidityBuffer.capacity() * 8 >= srcVector.getValueCount(), - "Not enough capacity for the validity buffer of the dst vector. " + - "Expected capacity %s, actual capacity %s", - (srcVector.getValueCount() + 7) / 8, dstValidityBuffer.capacity()); Preconditions.checkArgument( - dstOffsetBuffer.capacity() >= (srcVector.getValueCount() + 1) * ((long) BaseVariableWidthVector.OFFSET_WIDTH), - "Not enough capacity for the offset buffer of the dst vector. " + - "Expected capacity %s, actual capacity %s", - (srcVector.getValueCount() + 1) * BaseVariableWidthVector.OFFSET_WIDTH, dstOffsetBuffer.capacity()); - long dataSize = srcVector.getOffsetBuffer().getInt( - srcVector.getValueCount() * ((long) BaseVariableWidthVector.OFFSET_WIDTH)); + dstValidityBuffer.capacity() * 8 >= srcVector.getValueCount(), + "Not enough capacity for the validity buffer of the dst vector. " + + "Expected capacity %s, actual capacity %s", + (srcVector.getValueCount() + 7) / 8, + dstValidityBuffer.capacity()); + Preconditions.checkArgument( + dstOffsetBuffer.capacity() + >= (srcVector.getValueCount() + 1) * ((long) BaseVariableWidthVector.OFFSET_WIDTH), + "Not enough capacity for the offset buffer of the dst vector. " + + "Expected capacity %s, actual capacity %s", + (srcVector.getValueCount() + 1) * BaseVariableWidthVector.OFFSET_WIDTH, + dstOffsetBuffer.capacity()); + long dataSize = + srcVector + .getOffsetBuffer() + .getInt(srcVector.getValueCount() * ((long) BaseVariableWidthVector.OFFSET_WIDTH)); Preconditions.checkArgument( - dstValueBuffer.capacity() >= dataSize, "No enough capacity for the data buffer of the dst vector. " + - "Expected capacity %s, actual capacity %s", dataSize, dstValueBuffer.capacity()); + dstValueBuffer.capacity() >= dataSize, + "No enough capacity for the data buffer of the dst vector. " + + "Expected capacity %s, actual capacity %s", + dataSize, + dstValueBuffer.capacity()); // sort value indices try (IntVector sortedIndices = new IntVector("", srcVector.getAllocator())) { @@ -77,16 +86,19 @@ public void sortOutOfPlace(V srcVector, V dstVector, VectorValueComparator co BitVectorHelper.unsetBit(dstValidityBuffer, dstIndex); } else { BitVectorHelper.setBit(dstValidityBuffer, dstIndex); - int srcOffset = srcOffsetBuffer.getInt(srcIndex * ((long) BaseVariableWidthVector.OFFSET_WIDTH)); + int srcOffset = + srcOffsetBuffer.getInt(srcIndex * ((long) BaseVariableWidthVector.OFFSET_WIDTH)); int valueLength = - srcOffsetBuffer.getInt((srcIndex + 1) * ((long) BaseVariableWidthVector.OFFSET_WIDTH)) - srcOffset; + srcOffsetBuffer.getInt((srcIndex + 1) * ((long) BaseVariableWidthVector.OFFSET_WIDTH)) + - srcOffset; MemoryUtil.UNSAFE.copyMemory( - srcValueBuffer.memoryAddress() + srcOffset, - dstValueBuffer.memoryAddress() + dstOffset, - valueLength); + srcValueBuffer.memoryAddress() + srcOffset, + dstValueBuffer.memoryAddress() + dstOffset, + valueLength); dstOffset += valueLength; } - dstOffsetBuffer.setInt((dstIndex + 1) * ((long) BaseVariableWidthVector.OFFSET_WIDTH), dstOffset); + dstOffsetBuffer.setInt( + (dstIndex + 1) * ((long) BaseVariableWidthVector.OFFSET_WIDTH), dstOffset); } } } diff --git a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/VectorValueComparator.java b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/VectorValueComparator.java index d2c772ca8a819..0472f04109b1c 100644 --- a/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/VectorValueComparator.java +++ b/java/algorithm/src/main/java/org/apache/arrow/algorithm/sort/VectorValueComparator.java @@ -14,54 +14,44 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.sort; import org.apache.arrow.vector.ValueVector; /** - * Compare two values at the given indices in the vectors. - * This is used for vector sorting. + * Compare two values at the given indices in the vectors. This is used for vector sorting. + * * @param type of the vector. */ public abstract class VectorValueComparator { - /** - * The first vector to compare. - */ + /** The first vector to compare. */ protected V vector1; - /** - * The second vector to compare. - */ + /** The second vector to compare. */ protected V vector2; - /** - * Width of the vector value. For variable-length vectors, this value makes no sense. - */ + /** Width of the vector value. For variable-length vectors, this value makes no sense. */ protected int valueWidth; - private boolean checkNullsOnCompare = true; /** - * This value is true by default and re-computed when vectors are attached to the comparator. If both vectors cannot - * contain nulls then this value is {@code false} and calls to {@code compare(i1, i2)} are short-circuited - * to {@code compareNotNull(i1, i2)} thereby speeding up comparisons resulting in faster sorts etc. + * This value is true by default and re-computed when vectors are attached to the comparator. If + * both vectors cannot contain nulls then this value is {@code false} and calls to {@code + * compare(i1, i2)} are short-circuited to {@code compareNotNull(i1, i2)} thereby speeding up + * comparisons resulting in faster sorts etc. */ public boolean checkNullsOnCompare() { return this.checkNullsOnCompare; } - /** - * Constructor for variable-width vectors. - */ - protected VectorValueComparator() { - - } + /** Constructor for variable-width vectors. */ + protected VectorValueComparator() {} /** * Constructor for fixed-width vectors. + * * @param valueWidth the record width (in bytes). */ protected VectorValueComparator(int valueWidth) { @@ -74,6 +64,7 @@ public int getValueWidth() { /** * Attach both vectors to compare to the same input vector. + * * @param vector the vector to attach. */ public void attachVector(V vector) { @@ -82,6 +73,7 @@ public void attachVector(V vector) { /** * Attach vectors to compare. + * * @param vector1 the first vector to compare. * @param vector2 the second vector to compare. */ @@ -99,7 +91,7 @@ private boolean mayHaveNulls(V v) { if (v.getValueCount() == 0) { return true; } - if (! v.getField().isNullable()) { + if (!v.getField().isNullable()) { return false; } return v.getNullCount() > 0; @@ -107,11 +99,11 @@ private boolean mayHaveNulls(V v) { /** * Compare two values, given their indices. + * * @param index1 index of the first value to compare. * @param index2 index of the second value to compare. - * @return an integer greater than 0, if the first value is greater; - * an integer smaller than 0, if the first value is smaller; or 0, if both - * values are equal. + * @return an integer greater than 0, if the first value is greater; an integer smaller than 0, if + * the first value is smaller; or 0, if both values are equal. */ public int compare(int index1, int index2) { if (checkNullsOnCompare) { @@ -133,19 +125,19 @@ public int compare(int index1, int index2) { } /** - * Compare two values, given their indices. - * This is a fast path for comparing non-null values, so the caller - * must make sure that values at both indices are not null. + * Compare two values, given their indices. This is a fast path for comparing non-null values, so + * the caller must make sure that values at both indices are not null. + * * @param index1 index of the first value to compare. * @param index2 index of the second value to compare. - * @return an integer greater than 0, if the first value is greater; - * an integer smaller than 0, if the first value is smaller; or 0, if both - * values are equal. + * @return an integer greater than 0, if the first value is greater; an integer smaller than 0, if + * the first value is smaller; or 0, if both values are equal. */ public abstract int compareNotNull(int index1, int index2); /** * Creates a comparator of the same type. + * * @return the newly created comparator. */ public abstract VectorValueComparator createNew(); diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/deduplicate/TestDeduplicationUtils.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/deduplicate/TestDeduplicationUtils.java index ac083b84f1611..537189013a731 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/deduplicate/TestDeduplicationUtils.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/deduplicate/TestDeduplicationUtils.java @@ -14,14 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.deduplicate; import static org.junit.Assert.assertEquals; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import java.nio.charset.StandardCharsets; - import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; @@ -33,9 +31,7 @@ import org.junit.Before; import org.junit.Test; -/** - * Test cases for {@link DeduplicationUtils}. - */ +/** Test cases for {@link DeduplicationUtils}. */ public class TestDeduplicationUtils { private static final int VECTOR_LENGTH = 100; @@ -57,10 +53,11 @@ public void shutdown() { @Test public void testDeduplicateFixedWidth() { try (IntVector origVec = new IntVector("original vec", allocator); - IntVector dedupVec = new IntVector("deduplicated vec", allocator); - IntVector lengthVec = new IntVector("length vec", allocator); - ArrowBuf distinctBuf = allocator.buffer( - DataSizeRoundingUtil.divideBy8Ceil(VECTOR_LENGTH * REPETITION_COUNT))) { + IntVector dedupVec = new IntVector("deduplicated vec", allocator); + IntVector lengthVec = new IntVector("length vec", allocator); + ArrowBuf distinctBuf = + allocator.buffer( + DataSizeRoundingUtil.divideBy8Ceil(VECTOR_LENGTH * REPETITION_COUNT))) { origVec.allocateNew(VECTOR_LENGTH * REPETITION_COUNT); origVec.setValueCount(VECTOR_LENGTH * REPETITION_COUNT); lengthVec.allocateNew(); @@ -73,9 +70,10 @@ public void testDeduplicateFixedWidth() { } DeduplicationUtils.populateRunStartIndicators(origVec, distinctBuf); - assertEquals( VECTOR_LENGTH, - VECTOR_LENGTH * REPETITION_COUNT - - BitVectorHelper.getNullCount(distinctBuf, VECTOR_LENGTH * REPETITION_COUNT)); + assertEquals( + VECTOR_LENGTH, + VECTOR_LENGTH * REPETITION_COUNT + - BitVectorHelper.getNullCount(distinctBuf, VECTOR_LENGTH * REPETITION_COUNT)); DeduplicationUtils.populateDeduplicatedValues(distinctBuf, origVec, dedupVec); assertEquals(VECTOR_LENGTH, dedupVec.getValueCount()); @@ -84,7 +82,8 @@ public void testDeduplicateFixedWidth() { assertEquals(i, dedupVec.get(i)); } - DeduplicationUtils.populateRunLengths(distinctBuf, lengthVec, VECTOR_LENGTH * REPETITION_COUNT); + DeduplicationUtils.populateRunLengths( + distinctBuf, lengthVec, VECTOR_LENGTH * REPETITION_COUNT); assertEquals(VECTOR_LENGTH, lengthVec.getValueCount()); for (int i = 0; i < VECTOR_LENGTH; i++) { @@ -96,12 +95,12 @@ public void testDeduplicateFixedWidth() { @Test public void testDeduplicateVariableWidth() { try (VarCharVector origVec = new VarCharVector("original vec", allocator); - VarCharVector dedupVec = new VarCharVector("deduplicated vec", allocator); - IntVector lengthVec = new IntVector("length vec", allocator); - ArrowBuf distinctBuf = allocator.buffer( - DataSizeRoundingUtil.divideBy8Ceil(VECTOR_LENGTH * REPETITION_COUNT))) { - origVec.allocateNew( - VECTOR_LENGTH * REPETITION_COUNT * 10, VECTOR_LENGTH * REPETITION_COUNT); + VarCharVector dedupVec = new VarCharVector("deduplicated vec", allocator); + IntVector lengthVec = new IntVector("length vec", allocator); + ArrowBuf distinctBuf = + allocator.buffer( + DataSizeRoundingUtil.divideBy8Ceil(VECTOR_LENGTH * REPETITION_COUNT))) { + origVec.allocateNew(VECTOR_LENGTH * REPETITION_COUNT * 10, VECTOR_LENGTH * REPETITION_COUNT); origVec.setValueCount(VECTOR_LENGTH * REPETITION_COUNT); lengthVec.allocateNew(); @@ -114,9 +113,10 @@ public void testDeduplicateVariableWidth() { } DeduplicationUtils.populateRunStartIndicators(origVec, distinctBuf); - assertEquals(VECTOR_LENGTH, - VECTOR_LENGTH * REPETITION_COUNT - - BitVectorHelper.getNullCount(distinctBuf, VECTOR_LENGTH * REPETITION_COUNT)); + assertEquals( + VECTOR_LENGTH, + VECTOR_LENGTH * REPETITION_COUNT + - BitVectorHelper.getNullCount(distinctBuf, VECTOR_LENGTH * REPETITION_COUNT)); DeduplicationUtils.populateDeduplicatedValues(distinctBuf, origVec, dedupVec); assertEquals(VECTOR_LENGTH, dedupVec.getValueCount()); @@ -126,7 +126,7 @@ public void testDeduplicateVariableWidth() { } DeduplicationUtils.populateRunLengths( - distinctBuf, lengthVec, VECTOR_LENGTH * REPETITION_COUNT); + distinctBuf, lengthVec, VECTOR_LENGTH * REPETITION_COUNT); assertEquals(VECTOR_LENGTH, lengthVec.getValueCount()); for (int i = 0; i < VECTOR_LENGTH; i++) { diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/deduplicate/TestVectorRunDeduplicator.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/deduplicate/TestVectorRunDeduplicator.java index 788213b162870..820cadccae537 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/deduplicate/TestVectorRunDeduplicator.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/deduplicate/TestVectorRunDeduplicator.java @@ -14,14 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.deduplicate; import static org.junit.Assert.assertEquals; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import java.nio.charset.StandardCharsets; - import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.IntVector; @@ -30,9 +28,7 @@ import org.junit.Before; import org.junit.Test; -/** - * Test cases for {@link VectorRunDeduplicator}. - */ +/** Test cases for {@link VectorRunDeduplicator}. */ public class TestVectorRunDeduplicator { private static final int VECTOR_LENGTH = 100; @@ -57,7 +53,7 @@ public void testDeduplicateFixedWidth() { IntVector dedupVec = new IntVector("deduplicated vec", allocator); IntVector lengthVec = new IntVector("length vec", allocator); VectorRunDeduplicator deduplicator = - new VectorRunDeduplicator<>(origVec, allocator)) { + new VectorRunDeduplicator<>(origVec, allocator)) { origVec.allocateNew(VECTOR_LENGTH * REPETITION_COUNT); origVec.setValueCount(VECTOR_LENGTH * REPETITION_COUNT); lengthVec.allocateNew(); @@ -93,12 +89,11 @@ public void testDeduplicateFixedWidth() { @Test public void testDeduplicateVariableWidth() { try (VarCharVector origVec = new VarCharVector("original vec", allocator); - VarCharVector dedupVec = new VarCharVector("deduplicated vec", allocator); - IntVector lengthVec = new IntVector("length vec", allocator); - VectorRunDeduplicator deduplicator = - new VectorRunDeduplicator<>(origVec, allocator)) { - origVec.allocateNew( - VECTOR_LENGTH * REPETITION_COUNT * 10, VECTOR_LENGTH * REPETITION_COUNT); + VarCharVector dedupVec = new VarCharVector("deduplicated vec", allocator); + IntVector lengthVec = new IntVector("length vec", allocator); + VectorRunDeduplicator deduplicator = + new VectorRunDeduplicator<>(origVec, allocator)) { + origVec.allocateNew(VECTOR_LENGTH * REPETITION_COUNT * 10, VECTOR_LENGTH * REPETITION_COUNT); origVec.setValueCount(VECTOR_LENGTH * REPETITION_COUNT); lengthVec.allocateNew(); diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestHashTableBasedDictionaryBuilder.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestHashTableBasedDictionaryBuilder.java index 45c47626b720e..bfda86f26883d 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestHashTableBasedDictionaryBuilder.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestHashTableBasedDictionaryBuilder.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.dictionary; import static junit.framework.TestCase.assertTrue; @@ -23,7 +22,6 @@ import java.nio.charset.StandardCharsets; import java.util.Objects; - import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.IntVector; @@ -32,9 +30,7 @@ import org.junit.Before; import org.junit.Test; -/** - * Test cases for {@link HashTableBasedDictionaryBuilder}. - */ +/** Test cases for {@link HashTableBasedDictionaryBuilder}. */ public class TestHashTableBasedDictionaryBuilder { private BufferAllocator allocator; @@ -52,7 +48,7 @@ public void shutdown() { @Test public void testBuildVariableWidthDictionaryWithNull() { try (VarCharVector vec = new VarCharVector("", allocator); - VarCharVector dictionary = new VarCharVector("", allocator)) { + VarCharVector dictionary = new VarCharVector("", allocator)) { vec.allocateNew(100, 10); vec.setValueCount(10); @@ -72,27 +68,34 @@ public void testBuildVariableWidthDictionaryWithNull() { vec.set(9, "abc".getBytes(StandardCharsets.UTF_8)); HashTableBasedDictionaryBuilder dictionaryBuilder = - new HashTableBasedDictionaryBuilder<>(dictionary, true); + new HashTableBasedDictionaryBuilder<>(dictionary, true); int result = dictionaryBuilder.addValues(vec); assertEquals(7, result); assertEquals(7, dictionary.getValueCount()); - assertEquals("hello", new String(Objects.requireNonNull(dictionary.get(0)), StandardCharsets.UTF_8)); - assertEquals("abc", new String(Objects.requireNonNull(dictionary.get(1)), StandardCharsets.UTF_8)); + assertEquals( + "hello", new String(Objects.requireNonNull(dictionary.get(0)), StandardCharsets.UTF_8)); + assertEquals( + "abc", new String(Objects.requireNonNull(dictionary.get(1)), StandardCharsets.UTF_8)); assertNull(dictionary.get(2)); - assertEquals("world", new String(Objects.requireNonNull(dictionary.get(3)), StandardCharsets.UTF_8)); - assertEquals("12", new String(Objects.requireNonNull(dictionary.get(4)), StandardCharsets.UTF_8)); - assertEquals("dictionary", new String(Objects.requireNonNull(dictionary.get(5)), StandardCharsets.UTF_8)); - assertEquals("good", new String(Objects.requireNonNull(dictionary.get(6)), StandardCharsets.UTF_8)); + assertEquals( + "world", new String(Objects.requireNonNull(dictionary.get(3)), StandardCharsets.UTF_8)); + assertEquals( + "12", new String(Objects.requireNonNull(dictionary.get(4)), StandardCharsets.UTF_8)); + assertEquals( + "dictionary", + new String(Objects.requireNonNull(dictionary.get(5)), StandardCharsets.UTF_8)); + assertEquals( + "good", new String(Objects.requireNonNull(dictionary.get(6)), StandardCharsets.UTF_8)); } } @Test public void testBuildVariableWidthDictionaryWithoutNull() { try (VarCharVector vec = new VarCharVector("", allocator); - VarCharVector dictionary = new VarCharVector("", allocator)) { + VarCharVector dictionary = new VarCharVector("", allocator)) { vec.allocateNew(100, 10); vec.setValueCount(10); @@ -112,27 +115,33 @@ public void testBuildVariableWidthDictionaryWithoutNull() { vec.set(9, "abc".getBytes(StandardCharsets.UTF_8)); HashTableBasedDictionaryBuilder dictionaryBuilder = - new HashTableBasedDictionaryBuilder<>(dictionary, false); + new HashTableBasedDictionaryBuilder<>(dictionary, false); int result = dictionaryBuilder.addValues(vec); assertEquals(6, result); assertEquals(6, dictionary.getValueCount()); - assertEquals("hello", new String(Objects.requireNonNull(dictionary.get(0)), StandardCharsets.UTF_8)); - assertEquals("abc", new String(Objects.requireNonNull(dictionary.get(1)), StandardCharsets.UTF_8)); - assertEquals("world", new String(Objects.requireNonNull(dictionary.get(2)), StandardCharsets.UTF_8)); - assertEquals("12", new String(Objects.requireNonNull(dictionary.get(3)), StandardCharsets.UTF_8)); - assertEquals("dictionary", new String(Objects.requireNonNull(dictionary.get(4)), StandardCharsets.UTF_8)); - assertEquals("good", new String(Objects.requireNonNull(dictionary.get(5)), StandardCharsets.UTF_8)); - + assertEquals( + "hello", new String(Objects.requireNonNull(dictionary.get(0)), StandardCharsets.UTF_8)); + assertEquals( + "abc", new String(Objects.requireNonNull(dictionary.get(1)), StandardCharsets.UTF_8)); + assertEquals( + "world", new String(Objects.requireNonNull(dictionary.get(2)), StandardCharsets.UTF_8)); + assertEquals( + "12", new String(Objects.requireNonNull(dictionary.get(3)), StandardCharsets.UTF_8)); + assertEquals( + "dictionary", + new String(Objects.requireNonNull(dictionary.get(4)), StandardCharsets.UTF_8)); + assertEquals( + "good", new String(Objects.requireNonNull(dictionary.get(5)), StandardCharsets.UTF_8)); } } @Test public void testBuildFixedWidthDictionaryWithNull() { try (IntVector vec = new IntVector("", allocator); - IntVector dictionary = new IntVector("", allocator)) { + IntVector dictionary = new IntVector("", allocator)) { vec.allocateNew(10); vec.setValueCount(10); @@ -151,7 +160,7 @@ public void testBuildFixedWidthDictionaryWithNull() { vec.setNull(9); HashTableBasedDictionaryBuilder dictionaryBuilder = - new HashTableBasedDictionaryBuilder<>(dictionary, true); + new HashTableBasedDictionaryBuilder<>(dictionary, true); int result = dictionaryBuilder.addValues(vec); @@ -169,7 +178,7 @@ public void testBuildFixedWidthDictionaryWithNull() { @Test public void testBuildFixedWidthDictionaryWithoutNull() { try (IntVector vec = new IntVector("", allocator); - IntVector dictionary = new IntVector("", allocator)) { + IntVector dictionary = new IntVector("", allocator)) { vec.allocateNew(10); vec.setValueCount(10); @@ -188,7 +197,7 @@ public void testBuildFixedWidthDictionaryWithoutNull() { vec.setNull(9); HashTableBasedDictionaryBuilder dictionaryBuilder = - new HashTableBasedDictionaryBuilder<>(dictionary, false); + new HashTableBasedDictionaryBuilder<>(dictionary, false); int result = dictionaryBuilder.addValues(vec); @@ -199,7 +208,6 @@ public void testBuildFixedWidthDictionaryWithoutNull() { assertEquals(8, dictionary.get(1)); assertEquals(32, dictionary.get(2)); assertEquals(16, dictionary.get(3)); - } } } diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestHashTableDictionaryEncoder.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestHashTableDictionaryEncoder.java index 60efbf58bebda..b9646284a015b 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestHashTableDictionaryEncoder.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestHashTableDictionaryEncoder.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.dictionary; import static junit.framework.TestCase.assertTrue; @@ -25,7 +24,6 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Random; - import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.IntVector; @@ -38,9 +36,7 @@ import org.junit.Before; import org.junit.Test; -/** - * Test cases for {@link HashTableDictionaryEncoder}. - */ +/** Test cases for {@link HashTableDictionaryEncoder}. */ public class TestHashTableDictionaryEncoder { private final int VECTOR_LENGTH = 50; @@ -53,7 +49,7 @@ public class TestHashTableDictionaryEncoder { byte[] one = "111".getBytes(StandardCharsets.UTF_8); byte[] two = "222".getBytes(StandardCharsets.UTF_8); - byte[][] data = new byte[][]{zero, one, two}; + byte[][] data = new byte[][] {zero, one, two}; @Before public void prepare() { @@ -69,8 +65,8 @@ public void shutdown() { public void testEncodeAndDecode() { Random random = new Random(); try (VarCharVector rawVector = new VarCharVector("original vector", allocator); - IntVector encodedVector = new IntVector("encoded vector", allocator); - VarCharVector dictionary = new VarCharVector("dictionary", allocator)) { + IntVector encodedVector = new IntVector("encoded vector", allocator); + VarCharVector dictionary = new VarCharVector("dictionary", allocator)) { // set up dictionary dictionary.allocateNew(); @@ -89,7 +85,7 @@ public void testEncodeAndDecode() { rawVector.setValueCount(VECTOR_LENGTH); HashTableDictionaryEncoder encoder = - new HashTableDictionaryEncoder<>(dictionary, false); + new HashTableDictionaryEncoder<>(dictionary, false); // perform encoding encodedVector.allocateNew(); @@ -98,17 +94,21 @@ public void testEncodeAndDecode() { // verify encoding results assertEquals(rawVector.getValueCount(), encodedVector.getValueCount()); for (int i = 0; i < VECTOR_LENGTH; i++) { - assertArrayEquals(rawVector.get(i), String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8)); + assertArrayEquals( + rawVector.get(i), + String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8)); } // perform decoding Dictionary dict = new Dictionary(dictionary, new DictionaryEncoding(1L, false, null)); - try (VarCharVector decodedVector = (VarCharVector) DictionaryEncoder.decode(encodedVector, dict)) { + try (VarCharVector decodedVector = + (VarCharVector) DictionaryEncoder.decode(encodedVector, dict)) { // verify decoding results assertEquals(encodedVector.getValueCount(), decodedVector.getValueCount()); for (int i = 0; i < VECTOR_LENGTH; i++) { - assertArrayEquals(String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8), + assertArrayEquals( + String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8), decodedVector.get(i)); } } @@ -119,8 +119,8 @@ public void testEncodeAndDecode() { public void testEncodeAndDecodeWithNull() { Random random = new Random(); try (VarCharVector rawVector = new VarCharVector("original vector", allocator); - IntVector encodedVector = new IntVector("encoded vector", allocator); - VarCharVector dictionary = new VarCharVector("dictionary", allocator)) { + IntVector encodedVector = new IntVector("encoded vector", allocator); + VarCharVector dictionary = new VarCharVector("dictionary", allocator)) { // set up dictionary dictionary.allocateNew(); @@ -144,7 +144,7 @@ public void testEncodeAndDecodeWithNull() { rawVector.setValueCount(VECTOR_LENGTH); HashTableDictionaryEncoder encoder = - new HashTableDictionaryEncoder<>(dictionary, true); + new HashTableDictionaryEncoder<>(dictionary, true); // perform encoding encodedVector.allocateNew(); @@ -156,20 +156,24 @@ public void testEncodeAndDecodeWithNull() { if (i % 10 == 0) { assertEquals(0, encodedVector.get(i)); } else { - assertArrayEquals(rawVector.get(i), String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8)); + assertArrayEquals( + rawVector.get(i), + String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8)); } } // perform decoding Dictionary dict = new Dictionary(dictionary, new DictionaryEncoding(1L, false, null)); - try (VarCharVector decodedVector = (VarCharVector) DictionaryEncoder.decode(encodedVector, dict)) { + try (VarCharVector decodedVector = + (VarCharVector) DictionaryEncoder.decode(encodedVector, dict)) { // verify decoding results assertEquals(encodedVector.getValueCount(), decodedVector.getValueCount()); for (int i = 0; i < VECTOR_LENGTH; i++) { if (i % 10 == 0) { assertTrue(decodedVector.isNull(i)); } else { - assertArrayEquals(String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8), + assertArrayEquals( + String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8), decodedVector.get(i)); } } @@ -180,8 +184,8 @@ public void testEncodeAndDecodeWithNull() { @Test public void testEncodeNullWithoutNullInDictionary() { try (VarCharVector rawVector = new VarCharVector("original vector", allocator); - IntVector encodedVector = new IntVector("encoded vector", allocator); - VarCharVector dictionary = new VarCharVector("dictionary", allocator)) { + IntVector encodedVector = new IntVector("encoded vector", allocator); + VarCharVector dictionary = new VarCharVector("dictionary", allocator)) { // set up dictionary, with no null in it. dictionary.allocateNew(); @@ -199,13 +203,15 @@ public void testEncodeNullWithoutNullInDictionary() { encodedVector.allocateNew(); HashTableDictionaryEncoder encoder = - new HashTableDictionaryEncoder<>(dictionary, true); + new HashTableDictionaryEncoder<>(dictionary, true); // the encoder should encode null, but no null in the dictionary, // so an exception should be thrown. - assertThrows(IllegalArgumentException.class, () -> { - encoder.encode(rawVector, encodedVector); - }); + assertThrows( + IllegalArgumentException.class, + () -> { + encoder.encode(rawVector, encodedVector); + }); } } @@ -213,8 +219,8 @@ public void testEncodeNullWithoutNullInDictionary() { public void testEncodeStrings() { // Create a new value vector try (final VarCharVector vector = new VarCharVector("foo", allocator); - final IntVector encoded = new IntVector("encoded", allocator); - final VarCharVector dictionaryVector = new VarCharVector("dict", allocator)) { + final IntVector encoded = new IntVector("encoded", allocator); + final VarCharVector dictionaryVector = new VarCharVector("dict", allocator)) { vector.allocateNew(512, 5); encoded.allocateNew(); @@ -235,7 +241,7 @@ public void testEncodeStrings() { dictionaryVector.setValueCount(3); HashTableDictionaryEncoder encoder = - new HashTableDictionaryEncoder<>(dictionaryVector); + new HashTableDictionaryEncoder<>(dictionaryVector); encoder.encode(vector, encoded); // verify indices @@ -262,8 +268,8 @@ public void testEncodeStrings() { public void testEncodeLargeVector() { // Create a new value vector try (final VarCharVector vector = new VarCharVector("foo", allocator); - final IntVector encoded = new IntVector("encoded", allocator); - final VarCharVector dictionaryVector = new VarCharVector("dict", allocator)) { + final IntVector encoded = new IntVector("encoded", allocator); + final VarCharVector dictionaryVector = new VarCharVector("dict", allocator)) { vector.allocateNew(); encoded.allocateNew(); @@ -281,7 +287,7 @@ public void testEncodeLargeVector() { dictionaryVector.setValueCount(3); HashTableDictionaryEncoder encoder = - new HashTableDictionaryEncoder<>(dictionaryVector); + new HashTableDictionaryEncoder<>(dictionaryVector); encoder.encode(vector, encoded); assertEquals(count, encoded.getValueCount()); @@ -305,8 +311,8 @@ public void testEncodeLargeVector() { public void testEncodeBinaryVector() { // Create a new value vector try (final VarBinaryVector vector = new VarBinaryVector("foo", allocator); - final VarBinaryVector dictionaryVector = new VarBinaryVector("dict", allocator); - final IntVector encoded = new IntVector("encoded", allocator)) { + final VarBinaryVector dictionaryVector = new VarBinaryVector("dict", allocator); + final IntVector encoded = new IntVector("encoded", allocator)) { vector.allocateNew(512, 5); vector.allocateNew(); encoded.allocateNew(); @@ -327,7 +333,7 @@ public void testEncodeBinaryVector() { dictionaryVector.setValueCount(3); HashTableDictionaryEncoder encoder = - new HashTableDictionaryEncoder<>(dictionaryVector); + new HashTableDictionaryEncoder<>(dictionaryVector); encoder.encode(vector, encoded); assertEquals(5, encoded.getValueCount()); diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestLinearDictionaryEncoder.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestLinearDictionaryEncoder.java index a76aedffa308d..a4641704198cb 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestLinearDictionaryEncoder.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestLinearDictionaryEncoder.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.dictionary; import static junit.framework.TestCase.assertTrue; @@ -25,7 +24,6 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Random; - import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.IntVector; @@ -39,9 +37,7 @@ import org.junit.Before; import org.junit.Test; -/** - * Test cases for {@link LinearDictionaryEncoder}. - */ +/** Test cases for {@link LinearDictionaryEncoder}. */ public class TestLinearDictionaryEncoder { private final int VECTOR_LENGTH = 50; @@ -54,7 +50,7 @@ public class TestLinearDictionaryEncoder { byte[] one = "111".getBytes(StandardCharsets.UTF_8); byte[] two = "222".getBytes(StandardCharsets.UTF_8); - byte[][] data = new byte[][]{zero, one, two}; + byte[][] data = new byte[][] {zero, one, two}; @Before public void prepare() { @@ -70,8 +66,8 @@ public void shutdown() { public void testEncodeAndDecode() { Random random = new Random(); try (VarCharVector rawVector = new VarCharVector("original vector", allocator); - IntVector encodedVector = new IntVector("encoded vector", allocator); - VarCharVector dictionary = new VarCharVector("dictionary", allocator)) { + IntVector encodedVector = new IntVector("encoded vector", allocator); + VarCharVector dictionary = new VarCharVector("dictionary", allocator)) { // set up dictionary dictionary.allocateNew(); @@ -90,7 +86,7 @@ public void testEncodeAndDecode() { rawVector.setValueCount(VECTOR_LENGTH); LinearDictionaryEncoder encoder = - new LinearDictionaryEncoder<>(dictionary, false); + new LinearDictionaryEncoder<>(dictionary, false); // perform encoding encodedVector.allocateNew(); @@ -99,17 +95,21 @@ public void testEncodeAndDecode() { // verify encoding results assertEquals(rawVector.getValueCount(), encodedVector.getValueCount()); for (int i = 0; i < VECTOR_LENGTH; i++) { - assertArrayEquals(rawVector.get(i), String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8)); + assertArrayEquals( + rawVector.get(i), + String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8)); } // perform decoding Dictionary dict = new Dictionary(dictionary, new DictionaryEncoding(1L, false, null)); - try (VarCharVector decodedVector = (VarCharVector) DictionaryEncoder.decode(encodedVector, dict)) { + try (VarCharVector decodedVector = + (VarCharVector) DictionaryEncoder.decode(encodedVector, dict)) { // verify decoding results assertEquals(encodedVector.getValueCount(), decodedVector.getValueCount()); for (int i = 0; i < VECTOR_LENGTH; i++) { - assertArrayEquals(String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8), + assertArrayEquals( + String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8), decodedVector.get(i)); } } @@ -120,8 +120,8 @@ public void testEncodeAndDecode() { public void testEncodeAndDecodeWithNull() { Random random = new Random(); try (VarCharVector rawVector = new VarCharVector("original vector", allocator); - IntVector encodedVector = new IntVector("encoded vector", allocator); - VarCharVector dictionary = new VarCharVector("dictionary", allocator)) { + IntVector encodedVector = new IntVector("encoded vector", allocator); + VarCharVector dictionary = new VarCharVector("dictionary", allocator)) { // set up dictionary dictionary.allocateNew(); @@ -145,7 +145,7 @@ public void testEncodeAndDecodeWithNull() { rawVector.setValueCount(VECTOR_LENGTH); LinearDictionaryEncoder encoder = - new LinearDictionaryEncoder<>(dictionary, true); + new LinearDictionaryEncoder<>(dictionary, true); // perform encoding encodedVector.allocateNew(); @@ -157,13 +157,16 @@ public void testEncodeAndDecodeWithNull() { if (i % 10 == 0) { assertEquals(0, encodedVector.get(i)); } else { - assertArrayEquals(rawVector.get(i), String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8)); + assertArrayEquals( + rawVector.get(i), + String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8)); } } // perform decoding Dictionary dict = new Dictionary(dictionary, new DictionaryEncoding(1L, false, null)); - try (VarCharVector decodedVector = (VarCharVector) DictionaryEncoder.decode(encodedVector, dict)) { + try (VarCharVector decodedVector = + (VarCharVector) DictionaryEncoder.decode(encodedVector, dict)) { // verify decoding results assertEquals(encodedVector.getValueCount(), decodedVector.getValueCount()); @@ -171,7 +174,8 @@ public void testEncodeAndDecodeWithNull() { if (i % 10 == 0) { assertTrue(decodedVector.isNull(i)); } else { - assertArrayEquals(String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8), + assertArrayEquals( + String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8), decodedVector.get(i)); } } @@ -182,8 +186,8 @@ public void testEncodeAndDecodeWithNull() { @Test public void testEncodeNullWithoutNullInDictionary() { try (VarCharVector rawVector = new VarCharVector("original vector", allocator); - IntVector encodedVector = new IntVector("encoded vector", allocator); - VarCharVector dictionary = new VarCharVector("dictionary", allocator)) { + IntVector encodedVector = new IntVector("encoded vector", allocator); + VarCharVector dictionary = new VarCharVector("dictionary", allocator)) { // set up dictionary, with no null in it. dictionary.allocateNew(); @@ -201,13 +205,15 @@ public void testEncodeNullWithoutNullInDictionary() { encodedVector.allocateNew(); LinearDictionaryEncoder encoder = - new LinearDictionaryEncoder<>(dictionary, true); + new LinearDictionaryEncoder<>(dictionary, true); // the encoder should encode null, but no null in the dictionary, // so an exception should be thrown. - assertThrows(IllegalArgumentException.class, () -> { - encoder.encode(rawVector, encodedVector); - }); + assertThrows( + IllegalArgumentException.class, + () -> { + encoder.encode(rawVector, encodedVector); + }); } } @@ -215,8 +221,8 @@ public void testEncodeNullWithoutNullInDictionary() { public void testEncodeStrings() { // Create a new value vector try (final VarCharVector vector = new VarCharVector("foo", allocator); - final IntVector encoded = new IntVector("encoded", allocator); - final VarCharVector dictionaryVector = new VarCharVector("dict", allocator)) { + final IntVector encoded = new IntVector("encoded", allocator); + final VarCharVector dictionaryVector = new VarCharVector("dict", allocator)) { vector.allocateNew(512, 5); encoded.allocateNew(); @@ -237,7 +243,7 @@ public void testEncodeStrings() { dictionaryVector.setValueCount(3); LinearDictionaryEncoder encoder = - new LinearDictionaryEncoder<>(dictionaryVector); + new LinearDictionaryEncoder<>(dictionaryVector); encoder.encode(vector, encoded); // verify indices @@ -263,8 +269,8 @@ public void testEncodeStrings() { public void testEncodeLargeVector() { // Create a new value vector try (final VarCharVector vector = new VarCharVector("foo", allocator); - final IntVector encoded = new IntVector("encoded", allocator); - final VarCharVector dictionaryVector = new VarCharVector("dict", allocator)) { + final IntVector encoded = new IntVector("encoded", allocator); + final VarCharVector dictionaryVector = new VarCharVector("dict", allocator)) { vector.allocateNew(); encoded.allocateNew(); @@ -282,7 +288,7 @@ public void testEncodeLargeVector() { dictionaryVector.setValueCount(3); LinearDictionaryEncoder encoder = - new LinearDictionaryEncoder<>(dictionaryVector); + new LinearDictionaryEncoder<>(dictionaryVector); encoder.encode(vector, encoded); assertEquals(count, encoded.getValueCount()); @@ -306,8 +312,8 @@ public void testEncodeLargeVector() { public void testEncodeBinaryVector() { // Create a new value vector try (final VarBinaryVector vector = new VarBinaryVector("foo", allocator); - final VarBinaryVector dictionaryVector = new VarBinaryVector("dict", allocator); - final IntVector encoded = new IntVector("encoded", allocator)) { + final VarBinaryVector dictionaryVector = new VarBinaryVector("dict", allocator); + final IntVector encoded = new IntVector("encoded", allocator)) { vector.allocateNew(512, 5); vector.allocateNew(); encoded.allocateNew(); @@ -328,7 +334,7 @@ public void testEncodeBinaryVector() { dictionaryVector.setValueCount(3); LinearDictionaryEncoder encoder = - new LinearDictionaryEncoder<>(dictionaryVector); + new LinearDictionaryEncoder<>(dictionaryVector); encoder.encode(vector, encoded); assertEquals(5, encoded.getValueCount()); diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestSearchDictionaryEncoder.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestSearchDictionaryEncoder.java index e01c2e7905b46..e783e1f76818c 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestSearchDictionaryEncoder.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestSearchDictionaryEncoder.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.dictionary; import static junit.framework.TestCase.assertTrue; @@ -25,7 +24,6 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Random; - import org.apache.arrow.algorithm.sort.DefaultVectorComparators; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; @@ -40,9 +38,7 @@ import org.junit.Before; import org.junit.Test; -/** - * Test cases for {@link SearchDictionaryEncoder}. - */ +/** Test cases for {@link SearchDictionaryEncoder}. */ public class TestSearchDictionaryEncoder { private final int VECTOR_LENGTH = 50; @@ -55,7 +51,7 @@ public class TestSearchDictionaryEncoder { byte[] one = "111".getBytes(StandardCharsets.UTF_8); byte[] two = "222".getBytes(StandardCharsets.UTF_8); - byte[][] data = new byte[][]{zero, one, two}; + byte[][] data = new byte[][] {zero, one, two}; @Before public void prepare() { @@ -71,8 +67,8 @@ public void shutdown() { public void testEncodeAndDecode() { Random random = new Random(); try (VarCharVector rawVector = new VarCharVector("original vector", allocator); - IntVector encodedVector = new IntVector("encoded vector", allocator); - VarCharVector dictionary = new VarCharVector("dictionary", allocator)) { + IntVector encodedVector = new IntVector("encoded vector", allocator); + VarCharVector dictionary = new VarCharVector("dictionary", allocator)) { // set up dictionary dictionary.allocateNew(); @@ -91,8 +87,8 @@ public void testEncodeAndDecode() { rawVector.setValueCount(VECTOR_LENGTH); SearchDictionaryEncoder encoder = - new SearchDictionaryEncoder<>( - dictionary, DefaultVectorComparators.createDefaultComparator(rawVector), false); + new SearchDictionaryEncoder<>( + dictionary, DefaultVectorComparators.createDefaultComparator(rawVector), false); // perform encoding encodedVector.allocateNew(); @@ -101,17 +97,21 @@ public void testEncodeAndDecode() { // verify encoding results assertEquals(rawVector.getValueCount(), encodedVector.getValueCount()); for (int i = 0; i < VECTOR_LENGTH; i++) { - assertArrayEquals(rawVector.get(i), String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8)); + assertArrayEquals( + rawVector.get(i), + String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8)); } // perform decoding Dictionary dict = new Dictionary(dictionary, new DictionaryEncoding(1L, false, null)); - try (VarCharVector decodedVector = (VarCharVector) DictionaryEncoder.decode(encodedVector, dict)) { + try (VarCharVector decodedVector = + (VarCharVector) DictionaryEncoder.decode(encodedVector, dict)) { // verify decoding results assertEquals(encodedVector.getValueCount(), decodedVector.getValueCount()); for (int i = 0; i < VECTOR_LENGTH; i++) { - assertArrayEquals(String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8), + assertArrayEquals( + String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8), decodedVector.get(i)); } } @@ -122,8 +122,8 @@ public void testEncodeAndDecode() { public void testEncodeAndDecodeWithNull() { Random random = new Random(); try (VarCharVector rawVector = new VarCharVector("original vector", allocator); - IntVector encodedVector = new IntVector("encoded vector", allocator); - VarCharVector dictionary = new VarCharVector("dictionary", allocator)) { + IntVector encodedVector = new IntVector("encoded vector", allocator); + VarCharVector dictionary = new VarCharVector("dictionary", allocator)) { // set up dictionary dictionary.allocateNew(); @@ -147,8 +147,8 @@ public void testEncodeAndDecodeWithNull() { rawVector.setValueCount(VECTOR_LENGTH); SearchDictionaryEncoder encoder = - new SearchDictionaryEncoder<>( - dictionary, DefaultVectorComparators.createDefaultComparator(rawVector), true); + new SearchDictionaryEncoder<>( + dictionary, DefaultVectorComparators.createDefaultComparator(rawVector), true); // perform encoding encodedVector.allocateNew(); @@ -160,13 +160,16 @@ public void testEncodeAndDecodeWithNull() { if (i % 10 == 0) { assertEquals(0, encodedVector.get(i)); } else { - assertArrayEquals(rawVector.get(i), String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8)); + assertArrayEquals( + rawVector.get(i), + String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8)); } } // perform decoding Dictionary dict = new Dictionary(dictionary, new DictionaryEncoding(1L, false, null)); - try (VarCharVector decodedVector = (VarCharVector) DictionaryEncoder.decode(encodedVector, dict)) { + try (VarCharVector decodedVector = + (VarCharVector) DictionaryEncoder.decode(encodedVector, dict)) { // verify decoding results assertEquals(encodedVector.getValueCount(), decodedVector.getValueCount()); @@ -174,7 +177,8 @@ public void testEncodeAndDecodeWithNull() { if (i % 10 == 0) { assertTrue(decodedVector.isNull(i)); } else { - assertArrayEquals(String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8), + assertArrayEquals( + String.valueOf(encodedVector.get(i)).getBytes(StandardCharsets.UTF_8), decodedVector.get(i)); } } @@ -185,8 +189,8 @@ public void testEncodeAndDecodeWithNull() { @Test public void testEncodeNullWithoutNullInDictionary() { try (VarCharVector rawVector = new VarCharVector("original vector", allocator); - IntVector encodedVector = new IntVector("encoded vector", allocator); - VarCharVector dictionary = new VarCharVector("dictionary", allocator)) { + IntVector encodedVector = new IntVector("encoded vector", allocator); + VarCharVector dictionary = new VarCharVector("dictionary", allocator)) { // set up dictionary, with no null in it. dictionary.allocateNew(); @@ -204,14 +208,16 @@ public void testEncodeNullWithoutNullInDictionary() { encodedVector.allocateNew(); SearchDictionaryEncoder encoder = - new SearchDictionaryEncoder<>( - dictionary, DefaultVectorComparators.createDefaultComparator(rawVector), true); + new SearchDictionaryEncoder<>( + dictionary, DefaultVectorComparators.createDefaultComparator(rawVector), true); // the encoder should encode null, but no null in the dictionary, // so an exception should be thrown. - assertThrows(IllegalArgumentException.class, () -> { - encoder.encode(rawVector, encodedVector); - }); + assertThrows( + IllegalArgumentException.class, + () -> { + encoder.encode(rawVector, encodedVector); + }); } } @@ -219,8 +225,8 @@ public void testEncodeNullWithoutNullInDictionary() { public void testEncodeStrings() { // Create a new value vector try (final VarCharVector vector = new VarCharVector("foo", allocator); - final IntVector encoded = new IntVector("encoded", allocator); - final VarCharVector dictionaryVector = new VarCharVector("dict", allocator)) { + final IntVector encoded = new IntVector("encoded", allocator); + final VarCharVector dictionaryVector = new VarCharVector("dict", allocator)) { vector.allocateNew(512, 5); encoded.allocateNew(); @@ -241,8 +247,8 @@ public void testEncodeStrings() { dictionaryVector.setValueCount(3); SearchDictionaryEncoder encoder = - new SearchDictionaryEncoder<>( - dictionaryVector, DefaultVectorComparators.createDefaultComparator(vector)); + new SearchDictionaryEncoder<>( + dictionaryVector, DefaultVectorComparators.createDefaultComparator(vector)); encoder.encode(vector, encoded); // verify indices @@ -268,8 +274,8 @@ public void testEncodeStrings() { public void testEncodeLargeVector() { // Create a new value vector try (final VarCharVector vector = new VarCharVector("foo", allocator); - final IntVector encoded = new IntVector("encoded", allocator); - final VarCharVector dictionaryVector = new VarCharVector("dict", allocator)) { + final IntVector encoded = new IntVector("encoded", allocator); + final VarCharVector dictionaryVector = new VarCharVector("dict", allocator)) { vector.allocateNew(); encoded.allocateNew(); @@ -287,8 +293,8 @@ public void testEncodeLargeVector() { dictionaryVector.setValueCount(3); SearchDictionaryEncoder encoder = - new SearchDictionaryEncoder<>( - dictionaryVector, DefaultVectorComparators.createDefaultComparator(vector)); + new SearchDictionaryEncoder<>( + dictionaryVector, DefaultVectorComparators.createDefaultComparator(vector)); encoder.encode(vector, encoded); assertEquals(count, encoded.getValueCount()); @@ -312,8 +318,8 @@ public void testEncodeLargeVector() { public void testEncodeBinaryVector() { // Create a new value vector try (final VarBinaryVector vector = new VarBinaryVector("foo", allocator); - final VarBinaryVector dictionaryVector = new VarBinaryVector("dict", allocator); - final IntVector encoded = new IntVector("encoded", allocator)) { + final VarBinaryVector dictionaryVector = new VarBinaryVector("dict", allocator); + final IntVector encoded = new IntVector("encoded", allocator)) { vector.allocateNew(512, 5); vector.allocateNew(); encoded.allocateNew(); @@ -334,8 +340,8 @@ public void testEncodeBinaryVector() { dictionaryVector.setValueCount(3); SearchDictionaryEncoder encoder = - new SearchDictionaryEncoder<>( - dictionaryVector, DefaultVectorComparators.createDefaultComparator(vector)); + new SearchDictionaryEncoder<>( + dictionaryVector, DefaultVectorComparators.createDefaultComparator(vector)); encoder.encode(vector, encoded); assertEquals(5, encoded.getValueCount()); diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestSearchTreeBasedDictionaryBuilder.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestSearchTreeBasedDictionaryBuilder.java index 340b7e67e861f..6c8a57c1a4648 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestSearchTreeBasedDictionaryBuilder.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/dictionary/TestSearchTreeBasedDictionaryBuilder.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.dictionary; import static org.junit.Assert.assertEquals; @@ -22,7 +21,6 @@ import java.nio.charset.StandardCharsets; import java.util.Objects; - import org.apache.arrow.algorithm.sort.DefaultVectorComparators; import org.apache.arrow.algorithm.sort.VectorValueComparator; import org.apache.arrow.memory.BufferAllocator; @@ -33,9 +31,7 @@ import org.junit.Before; import org.junit.Test; -/** - * Test cases for {@link SearchTreeBasedDictionaryBuilder}. - */ +/** Test cases for {@link SearchTreeBasedDictionaryBuilder}. */ public class TestSearchTreeBasedDictionaryBuilder { private BufferAllocator allocator; @@ -53,8 +49,8 @@ public void shutdown() { @Test public void testBuildVariableWidthDictionaryWithNull() { try (VarCharVector vec = new VarCharVector("", allocator); - VarCharVector dictionary = new VarCharVector("", allocator); - VarCharVector sortedDictionary = new VarCharVector("", allocator)) { + VarCharVector dictionary = new VarCharVector("", allocator); + VarCharVector sortedDictionary = new VarCharVector("", allocator)) { vec.allocateNew(100, 10); vec.setValueCount(10); @@ -74,9 +70,10 @@ public void testBuildVariableWidthDictionaryWithNull() { vec.set(8, "good".getBytes(StandardCharsets.UTF_8)); vec.set(9, "abc".getBytes(StandardCharsets.UTF_8)); - VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vec); SearchTreeBasedDictionaryBuilder dictionaryBuilder = - new SearchTreeBasedDictionaryBuilder<>(dictionary, comparator, true); + new SearchTreeBasedDictionaryBuilder<>(dictionary, comparator, true); int result = dictionaryBuilder.addValues(vec); @@ -86,20 +83,32 @@ public void testBuildVariableWidthDictionaryWithNull() { dictionaryBuilder.populateSortedDictionary(sortedDictionary); assertTrue(sortedDictionary.isNull(0)); - assertEquals("12", new String(Objects.requireNonNull(sortedDictionary.get(1)), StandardCharsets.UTF_8)); - assertEquals("abc", new String(Objects.requireNonNull(sortedDictionary.get(2)), StandardCharsets.UTF_8)); - assertEquals("dictionary", new String(Objects.requireNonNull(sortedDictionary.get(3)), StandardCharsets.UTF_8)); - assertEquals("good", new String(Objects.requireNonNull(sortedDictionary.get(4)), StandardCharsets.UTF_8)); - assertEquals("hello", new String(Objects.requireNonNull(sortedDictionary.get(5)), StandardCharsets.UTF_8)); - assertEquals("world", new String(Objects.requireNonNull(sortedDictionary.get(6)), StandardCharsets.UTF_8)); + assertEquals( + "12", + new String(Objects.requireNonNull(sortedDictionary.get(1)), StandardCharsets.UTF_8)); + assertEquals( + "abc", + new String(Objects.requireNonNull(sortedDictionary.get(2)), StandardCharsets.UTF_8)); + assertEquals( + "dictionary", + new String(Objects.requireNonNull(sortedDictionary.get(3)), StandardCharsets.UTF_8)); + assertEquals( + "good", + new String(Objects.requireNonNull(sortedDictionary.get(4)), StandardCharsets.UTF_8)); + assertEquals( + "hello", + new String(Objects.requireNonNull(sortedDictionary.get(5)), StandardCharsets.UTF_8)); + assertEquals( + "world", + new String(Objects.requireNonNull(sortedDictionary.get(6)), StandardCharsets.UTF_8)); } } @Test public void testBuildVariableWidthDictionaryWithoutNull() { try (VarCharVector vec = new VarCharVector("", allocator); - VarCharVector dictionary = new VarCharVector("", allocator); - VarCharVector sortedDictionary = new VarCharVector("", allocator)) { + VarCharVector dictionary = new VarCharVector("", allocator); + VarCharVector sortedDictionary = new VarCharVector("", allocator)) { vec.allocateNew(100, 10); vec.setValueCount(10); @@ -119,9 +128,10 @@ public void testBuildVariableWidthDictionaryWithoutNull() { vec.set(8, "good".getBytes(StandardCharsets.UTF_8)); vec.set(9, "abc".getBytes(StandardCharsets.UTF_8)); - VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vec); SearchTreeBasedDictionaryBuilder dictionaryBuilder = - new SearchTreeBasedDictionaryBuilder<>(dictionary, comparator, false); + new SearchTreeBasedDictionaryBuilder<>(dictionary, comparator, false); int result = dictionaryBuilder.addValues(vec); @@ -130,20 +140,32 @@ public void testBuildVariableWidthDictionaryWithoutNull() { dictionaryBuilder.populateSortedDictionary(sortedDictionary); - assertEquals("12", new String(Objects.requireNonNull(sortedDictionary.get(0)), StandardCharsets.UTF_8)); - assertEquals("abc", new String(Objects.requireNonNull(sortedDictionary.get(1)), StandardCharsets.UTF_8)); - assertEquals("dictionary", new String(Objects.requireNonNull(sortedDictionary.get(2)), StandardCharsets.UTF_8)); - assertEquals("good", new String(Objects.requireNonNull(sortedDictionary.get(3)), StandardCharsets.UTF_8)); - assertEquals("hello", new String(Objects.requireNonNull(sortedDictionary.get(4)), StandardCharsets.UTF_8)); - assertEquals("world", new String(Objects.requireNonNull(sortedDictionary.get(5)), StandardCharsets.UTF_8)); + assertEquals( + "12", + new String(Objects.requireNonNull(sortedDictionary.get(0)), StandardCharsets.UTF_8)); + assertEquals( + "abc", + new String(Objects.requireNonNull(sortedDictionary.get(1)), StandardCharsets.UTF_8)); + assertEquals( + "dictionary", + new String(Objects.requireNonNull(sortedDictionary.get(2)), StandardCharsets.UTF_8)); + assertEquals( + "good", + new String(Objects.requireNonNull(sortedDictionary.get(3)), StandardCharsets.UTF_8)); + assertEquals( + "hello", + new String(Objects.requireNonNull(sortedDictionary.get(4)), StandardCharsets.UTF_8)); + assertEquals( + "world", + new String(Objects.requireNonNull(sortedDictionary.get(5)), StandardCharsets.UTF_8)); } } @Test public void testBuildFixedWidthDictionaryWithNull() { try (IntVector vec = new IntVector("", allocator); - IntVector dictionary = new IntVector("", allocator); - IntVector sortedDictionary = new IntVector("", allocator)) { + IntVector dictionary = new IntVector("", allocator); + IntVector sortedDictionary = new IntVector("", allocator)) { vec.allocateNew(10); vec.setValueCount(10); @@ -162,9 +184,10 @@ public void testBuildFixedWidthDictionaryWithNull() { vec.set(8, 4); vec.setNull(9); - VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vec); SearchTreeBasedDictionaryBuilder dictionaryBuilder = - new SearchTreeBasedDictionaryBuilder<>(dictionary, comparator, true); + new SearchTreeBasedDictionaryBuilder<>(dictionary, comparator, true); int result = dictionaryBuilder.addValues(vec); @@ -184,8 +207,8 @@ public void testBuildFixedWidthDictionaryWithNull() { @Test public void testBuildFixedWidthDictionaryWithoutNull() { try (IntVector vec = new IntVector("", allocator); - IntVector dictionary = new IntVector("", allocator); - IntVector sortedDictionary = new IntVector("", allocator)) { + IntVector dictionary = new IntVector("", allocator); + IntVector sortedDictionary = new IntVector("", allocator)) { vec.allocateNew(10); vec.setValueCount(10); @@ -204,9 +227,10 @@ public void testBuildFixedWidthDictionaryWithoutNull() { vec.set(8, 4); vec.setNull(9); - VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vec); SearchTreeBasedDictionaryBuilder dictionaryBuilder = - new SearchTreeBasedDictionaryBuilder<>(dictionary, comparator, false); + new SearchTreeBasedDictionaryBuilder<>(dictionary, comparator, false); int result = dictionaryBuilder.addValues(vec); diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/misc/TestPartialSumUtils.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/misc/TestPartialSumUtils.java index 630dd80b44084..e3ab981670e9e 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/misc/TestPartialSumUtils.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/misc/TestPartialSumUtils.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.misc; import static org.junit.Assert.assertEquals; @@ -26,9 +25,7 @@ import org.junit.Before; import org.junit.Test; -/** - * Test cases for {@link PartialSumUtils}. - */ +/** Test cases for {@link PartialSumUtils}. */ public class TestPartialSumUtils { private static final int PARTIAL_SUM_VECTOR_LENGTH = 101; @@ -50,7 +47,7 @@ public void shutdown() { @Test public void testToPartialSumVector() { try (IntVector delta = new IntVector("delta", allocator); - IntVector partialSum = new IntVector("partial sum", allocator)) { + IntVector partialSum = new IntVector("partial sum", allocator)) { delta.allocateNew(DELTA_VECTOR_LENGTH); delta.setValueCount(DELTA_VECTOR_LENGTH); @@ -75,7 +72,7 @@ public void testToPartialSumVector() { @Test public void testToDeltaVector() { try (IntVector partialSum = new IntVector("partial sum", allocator); - IntVector delta = new IntVector("delta", allocator)) { + IntVector delta = new IntVector("delta", allocator)) { partialSum.allocateNew(PARTIAL_SUM_VECTOR_LENGTH); partialSum.setValueCount(PARTIAL_SUM_VECTOR_LENGTH); @@ -111,7 +108,8 @@ public void testFindPositionInPartialSumVector() { // search and verify results for (int i = 0; i < PARTIAL_SUM_VECTOR_LENGTH - 1; i++) { - assertEquals(i, PartialSumUtils.findPositionInPartialSumVector(partialSum, sumBase + 3 * i + 1)); + assertEquals( + i, PartialSumUtils.findPositionInPartialSumVector(partialSum, sumBase + 3 * i + 1)); } } } @@ -131,8 +129,10 @@ public void testFindPositionInPartialSumVectorNegative() { // search and verify results assertEquals(0, PartialSumUtils.findPositionInPartialSumVector(partialSum, sumBase)); assertEquals(-1, PartialSumUtils.findPositionInPartialSumVector(partialSum, sumBase - 1)); - assertEquals(-1, PartialSumUtils.findPositionInPartialSumVector(partialSum, - sumBase + 3 * (PARTIAL_SUM_VECTOR_LENGTH - 1))); + assertEquals( + -1, + PartialSumUtils.findPositionInPartialSumVector( + partialSum, sumBase + 3 * (PARTIAL_SUM_VECTOR_LENGTH - 1))); } } } diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/rank/TestVectorRank.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/rank/TestVectorRank.java index 0e6627eb4822a..4b7c6a9756780 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/rank/TestVectorRank.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/rank/TestVectorRank.java @@ -14,14 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.rank; import static org.junit.Assert.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import java.nio.charset.StandardCharsets; - import org.apache.arrow.algorithm.sort.DefaultVectorComparators; import org.apache.arrow.algorithm.sort.VectorValueComparator; import org.apache.arrow.memory.BufferAllocator; @@ -32,9 +30,7 @@ import org.junit.Before; import org.junit.Test; -/** - * Test cases for {@link org.apache.arrow.algorithm.rank.VectorRank}. - */ +/** Test cases for {@link org.apache.arrow.algorithm.rank.VectorRank}. */ public class TestVectorRank { private BufferAllocator allocator; @@ -70,7 +66,7 @@ public void testFixedWidthRank() { vector.set(9, 6); VectorValueComparator comparator = - DefaultVectorComparators.createDefaultComparator(vector); + DefaultVectorComparators.createDefaultComparator(vector); assertEquals(7, rank.indexAtRank(vector, comparator, 0)); assertEquals(0, rank.indexAtRank(vector, comparator, 1)); assertEquals(6, rank.indexAtRank(vector, comparator, 2)); @@ -103,7 +99,7 @@ public void testVariableWidthRank() { vector.set(9, String.valueOf(6).getBytes(StandardCharsets.UTF_8)); VectorValueComparator comparator = - DefaultVectorComparators.createDefaultComparator(vector); + DefaultVectorComparators.createDefaultComparator(vector); assertEquals(7, rank.indexAtRank(vector, comparator, 0)); assertEquals(0, rank.indexAtRank(vector, comparator, 1)); @@ -137,11 +133,13 @@ public void testRankNegative() { vector.set(9, 6); VectorValueComparator comparator = - DefaultVectorComparators.createDefaultComparator(vector); + DefaultVectorComparators.createDefaultComparator(vector); - assertThrows(IllegalArgumentException.class, () -> { - rank.indexAtRank(vector, comparator, VECTOR_LENGTH + 1); - }); + assertThrows( + IllegalArgumentException.class, + () -> { + rank.indexAtRank(vector, comparator, VECTOR_LENGTH + 1); + }); } } } diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/search/TestParallelSearcher.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/search/TestParallelSearcher.java index 9ccecfa84a73a..7ff86a743effd 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/search/TestParallelSearcher.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/search/TestParallelSearcher.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.search; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -26,7 +25,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; - import org.apache.arrow.algorithm.sort.DefaultVectorComparators; import org.apache.arrow.algorithm.sort.VectorValueComparator; import org.apache.arrow.memory.BufferAllocator; @@ -39,9 +37,7 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -/** - * Test cases for {@link ParallelSearcher}. - */ +/** Test cases for {@link ParallelSearcher}. */ @RunWith(Parameterized.class) public class TestParallelSearcher { @@ -97,8 +93,10 @@ public void testParallelIntSearch() throws ExecutionException, InterruptedExcept keyVector.allocateNew(VECTOR_LENGTH); // if we are comparing elements using equality semantics, we do not need a comparator here. - VectorValueComparator comparator = comparatorType == ComparatorType.EqualityComparator ? null - : DefaultVectorComparators.createDefaultComparator(targetVector); + VectorValueComparator comparator = + comparatorType == ComparatorType.EqualityComparator + ? null + : DefaultVectorComparators.createDefaultComparator(targetVector); for (int i = 0; i < VECTOR_LENGTH; i++) { targetVector.set(i, i); @@ -107,9 +105,13 @@ public void testParallelIntSearch() throws ExecutionException, InterruptedExcept targetVector.setValueCount(VECTOR_LENGTH); keyVector.setValueCount(VECTOR_LENGTH); - ParallelSearcher searcher = new ParallelSearcher<>(targetVector, threadPool, threadCount); + ParallelSearcher searcher = + new ParallelSearcher<>(targetVector, threadPool, threadCount); for (int i = 0; i < VECTOR_LENGTH; i++) { - int pos = comparator == null ? searcher.search(keyVector, i) : searcher.search(keyVector, i, comparator); + int pos = + comparator == null + ? searcher.search(keyVector, i) + : searcher.search(keyVector, i, comparator); if (i * 2 < VECTOR_LENGTH) { assertEquals(i * 2, pos); } else { @@ -122,13 +124,15 @@ public void testParallelIntSearch() throws ExecutionException, InterruptedExcept @Test public void testParallelStringSearch() throws ExecutionException, InterruptedException { try (VarCharVector targetVector = new VarCharVector("targetVector", allocator); - VarCharVector keyVector = new VarCharVector("keyVector", allocator)) { + VarCharVector keyVector = new VarCharVector("keyVector", allocator)) { targetVector.allocateNew(VECTOR_LENGTH); keyVector.allocateNew(VECTOR_LENGTH); // if we are comparing elements using equality semantics, we do not need a comparator here. - VectorValueComparator comparator = comparatorType == ComparatorType.EqualityComparator ? null - : DefaultVectorComparators.createDefaultComparator(targetVector); + VectorValueComparator comparator = + comparatorType == ComparatorType.EqualityComparator + ? null + : DefaultVectorComparators.createDefaultComparator(targetVector); for (int i = 0; i < VECTOR_LENGTH; i++) { targetVector.setSafe(i, String.valueOf(i).getBytes(StandardCharsets.UTF_8)); @@ -137,9 +141,13 @@ public void testParallelStringSearch() throws ExecutionException, InterruptedExc targetVector.setValueCount(VECTOR_LENGTH); keyVector.setValueCount(VECTOR_LENGTH); - ParallelSearcher searcher = new ParallelSearcher<>(targetVector, threadPool, threadCount); + ParallelSearcher searcher = + new ParallelSearcher<>(targetVector, threadPool, threadCount); for (int i = 0; i < VECTOR_LENGTH; i++) { - int pos = comparator == null ? searcher.search(keyVector, i) : searcher.search(keyVector, i, comparator); + int pos = + comparator == null + ? searcher.search(keyVector, i) + : searcher.search(keyVector, i, comparator); if (i * 2 < VECTOR_LENGTH) { assertEquals(i * 2, pos); } else { diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/search/TestVectorRangeSearcher.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/search/TestVectorRangeSearcher.java index 18f4fa0355f4f..39f2f609f7df4 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/search/TestVectorRangeSearcher.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/search/TestVectorRangeSearcher.java @@ -14,14 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.search; import static org.junit.Assert.assertEquals; import java.util.Arrays; import java.util.Collection; - import org.apache.arrow.algorithm.sort.DefaultVectorComparators; import org.apache.arrow.algorithm.sort.VectorValueComparator; import org.apache.arrow.memory.BufferAllocator; @@ -33,9 +31,7 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -/** - * Test cases for {@link VectorRangeSearcher}. - */ +/** Test cases for {@link VectorRangeSearcher}. */ @RunWith(Parameterized.class) public class TestVectorRangeSearcher { @@ -78,9 +74,11 @@ public void testGetLowerBounds() { } // do search - VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(intVector); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(intVector); for (int i = 0; i < maxValue; i++) { - int result = VectorRangeSearcher.getFirstMatch(intVector, comparator, intVector, i * repeat); + int result = + VectorRangeSearcher.getFirstMatch(intVector, comparator, intVector, i * repeat); assertEquals(i * ((long) repeat), result); } } @@ -112,7 +110,8 @@ public void testGetLowerBoundsNegative() { } // do search - VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(intVector); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(intVector); for (int i = 0; i < maxValue; i++) { int result = VectorRangeSearcher.getFirstMatch(intVector, comparator, negVector, i); assertEquals(-1, result); @@ -141,7 +140,8 @@ public void testGetUpperBounds() { } // do search - VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(intVector); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(intVector); for (int i = 0; i < maxValue; i++) { int result = VectorRangeSearcher.getLastMatch(intVector, comparator, intVector, i * repeat); assertEquals((i + 1) * repeat - 1, result); @@ -153,7 +153,7 @@ public void testGetUpperBounds() { public void testGetUpperBoundsNegative() { final int maxValue = 100; try (IntVector intVector = new IntVector("int vec", allocator); - IntVector negVector = new IntVector("neg vec", allocator)) { + IntVector negVector = new IntVector("neg vec", allocator)) { // allocate vector intVector.allocateNew(maxValue * repeat); intVector.setValueCount(maxValue * repeat); @@ -175,7 +175,8 @@ public void testGetUpperBoundsNegative() { } // do search - VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(intVector); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(intVector); for (int i = 0; i < maxValue; i++) { int result = VectorRangeSearcher.getLastMatch(intVector, comparator, negVector, i); assertEquals(-1, result); @@ -185,11 +186,6 @@ public void testGetUpperBoundsNegative() { @Parameterized.Parameters(name = "repeat = {0}") public static Collection getRepeat() { - return Arrays.asList( - new Object[]{1}, - new Object[]{2}, - new Object[]{5}, - new Object[]{10} - ); + return Arrays.asList(new Object[] {1}, new Object[] {2}, new Object[] {5}, new Object[] {10}); } } diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/search/TestVectorSearcher.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/search/TestVectorSearcher.java index 32fa10bbd98d0..629d900b479b6 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/search/TestVectorSearcher.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/search/TestVectorSearcher.java @@ -14,14 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.search; import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.OFFSET_WIDTH; import static org.junit.Assert.assertEquals; import java.nio.charset.StandardCharsets; - import org.apache.arrow.algorithm.sort.DefaultVectorComparators; import org.apache.arrow.algorithm.sort.VectorValueComparator; import org.apache.arrow.memory.BufferAllocator; @@ -37,9 +35,7 @@ import org.junit.Before; import org.junit.Test; -/** - * Test cases for {@link org.apache.arrow.algorithm.search.VectorSearcher}. - */ +/** Test cases for {@link org.apache.arrow.algorithm.search.VectorSearcher}. */ public class TestVectorSearcher { private final int VECTOR_LENGTH = 100; @@ -59,7 +55,7 @@ public void shutdown() { @Test public void testBinarySearchInt() { try (IntVector rawVector = new IntVector("", allocator); - IntVector negVector = new IntVector("", allocator)) { + IntVector negVector = new IntVector("", allocator)) { rawVector.allocateNew(VECTOR_LENGTH); rawVector.setValueCount(VECTOR_LENGTH); negVector.allocateNew(1); @@ -77,7 +73,7 @@ public void testBinarySearchInt() { // do search VectorValueComparator comparator = - DefaultVectorComparators.createDefaultComparator(rawVector); + DefaultVectorComparators.createDefaultComparator(rawVector); for (int i = 0; i < VECTOR_LENGTH; i++) { int result = VectorSearcher.binarySearch(rawVector, comparator, rawVector, i); assertEquals(i, result); @@ -91,7 +87,7 @@ public void testBinarySearchInt() { @Test public void testLinearSearchInt() { try (IntVector rawVector = new IntVector("", allocator); - IntVector negVector = new IntVector("", allocator)) { + IntVector negVector = new IntVector("", allocator)) { rawVector.allocateNew(VECTOR_LENGTH); rawVector.setValueCount(VECTOR_LENGTH); negVector.allocateNew(1); @@ -109,7 +105,7 @@ public void testLinearSearchInt() { // do search VectorValueComparator comparator = - DefaultVectorComparators.createDefaultComparator(rawVector); + DefaultVectorComparators.createDefaultComparator(rawVector); for (int i = 0; i < VECTOR_LENGTH; i++) { int result = VectorSearcher.linearSearch(rawVector, comparator, rawVector, i); assertEquals(i, result); @@ -123,7 +119,7 @@ public void testLinearSearchInt() { @Test public void testBinarySearchVarChar() { try (VarCharVector rawVector = new VarCharVector("", allocator); - VarCharVector negVector = new VarCharVector("", allocator)) { + VarCharVector negVector = new VarCharVector("", allocator)) { rawVector.allocateNew(VECTOR_LENGTH * 16, VECTOR_LENGTH); rawVector.setValueCount(VECTOR_LENGTH); negVector.allocateNew(VECTOR_LENGTH, 1); @@ -148,7 +144,7 @@ public void testBinarySearchVarChar() { // do search VectorValueComparator comparator = - DefaultVectorComparators.createDefaultComparator(rawVector); + DefaultVectorComparators.createDefaultComparator(rawVector); for (int i = 0; i < VECTOR_LENGTH; i++) { int result = VectorSearcher.binarySearch(rawVector, comparator, rawVector, i); assertEquals(i, result); @@ -162,7 +158,7 @@ public void testBinarySearchVarChar() { @Test public void testLinearSearchVarChar() { try (VarCharVector rawVector = new VarCharVector("", allocator); - VarCharVector negVector = new VarCharVector("", allocator)) { + VarCharVector negVector = new VarCharVector("", allocator)) { rawVector.allocateNew(VECTOR_LENGTH * 16, VECTOR_LENGTH); rawVector.setValueCount(VECTOR_LENGTH); negVector.allocateNew(VECTOR_LENGTH, 1); @@ -187,7 +183,7 @@ public void testLinearSearchVarChar() { // do search VectorValueComparator comparator = - DefaultVectorComparators.createDefaultComparator(rawVector); + DefaultVectorComparators.createDefaultComparator(rawVector); for (int i = 0; i < VECTOR_LENGTH; i++) { int result = VectorSearcher.linearSearch(rawVector, comparator, rawVector, i); assertEquals(i, result); @@ -260,11 +256,11 @@ private ListVector createNegativeListVector() { @Test public void testBinarySearchList() { try (ListVector rawVector = createListVector(); - ListVector negVector = createNegativeListVector()) { + ListVector negVector = createNegativeListVector()) { // do search VectorValueComparator comparator = - DefaultVectorComparators.createDefaultComparator(rawVector); + DefaultVectorComparators.createDefaultComparator(rawVector); for (int i = 0; i < rawVector.getValueCount(); i++) { int result = VectorSearcher.binarySearch(rawVector, comparator, rawVector, i); assertEquals(i, result); @@ -281,11 +277,11 @@ public void testBinarySearchList() { @Test public void testLinearSearchList() { try (ListVector rawVector = createListVector(); - ListVector negVector = createNegativeListVector()) { + ListVector negVector = createNegativeListVector()) { // do search VectorValueComparator comparator = - DefaultVectorComparators.createDefaultComparator(rawVector); + DefaultVectorComparators.createDefaultComparator(rawVector); for (int i = 0; i < rawVector.getValueCount(); i++) { int result = VectorSearcher.linearSearch(rawVector, comparator, rawVector, i); assertEquals(i, result); diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestCompositeVectorComparator.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestCompositeVectorComparator.java index 9624432924b5a..21f6c0217c376 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestCompositeVectorComparator.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestCompositeVectorComparator.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.sort; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -22,7 +21,6 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; - import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.IntVector; @@ -33,9 +31,7 @@ import org.junit.Before; import org.junit.Test; -/** - * Test cases for {@link CompositeVectorComparator}. - */ +/** Test cases for {@link CompositeVectorComparator}. */ public class TestCompositeVectorComparator { private BufferAllocator allocator; @@ -60,7 +56,7 @@ public void testCompareVectorSchemaRoot() { VarCharVector strVec2 = new VarCharVector("str2", allocator); try (VectorSchemaRoot batch1 = new VectorSchemaRoot(Arrays.asList(intVec1, strVec1)); - VectorSchemaRoot batch2 = new VectorSchemaRoot(Arrays.asList(intVec2, strVec2))) { + VectorSchemaRoot batch2 = new VectorSchemaRoot(Arrays.asList(intVec2, strVec2))) { intVec1.allocateNew(vectorLength); strVec1.allocateNew(vectorLength * 10, vectorLength); @@ -75,15 +71,15 @@ public void testCompareVectorSchemaRoot() { } VectorValueComparator innerComparator1 = - DefaultVectorComparators.createDefaultComparator(intVec1); + DefaultVectorComparators.createDefaultComparator(intVec1); innerComparator1.attachVectors(intVec1, intVec2); VectorValueComparator innerComparator2 = - DefaultVectorComparators.createDefaultComparator(strVec1); + DefaultVectorComparators.createDefaultComparator(strVec1); innerComparator2.attachVectors(strVec1, strVec2); - VectorValueComparator comparator = new CompositeVectorComparator( - new VectorValueComparator[]{innerComparator1, innerComparator2} - ); + VectorValueComparator comparator = + new CompositeVectorComparator( + new VectorValueComparator[] {innerComparator1, innerComparator2}); // verify results diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestDefaultVectorComparator.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestDefaultVectorComparator.java index c40854fb17410..f1b3d6fb5aa1d 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestDefaultVectorComparator.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestDefaultVectorComparator.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.sort; import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.OFFSET_WIDTH; @@ -67,9 +66,7 @@ import org.junit.Test; import org.junit.jupiter.api.Assertions; -/** - * Test cases for {@link DefaultVectorComparators}. - */ +/** Test cases for {@link DefaultVectorComparators}. */ public class TestDefaultVectorComparator { private BufferAllocator allocator; @@ -111,9 +108,9 @@ private ListVector createListVector(int count) { @Test public void testCompareLists() { try (ListVector listVector1 = createListVector(10); - ListVector listVector2 = createListVector(11)) { + ListVector listVector2 = createListVector(11)) { VectorValueComparator comparator = - DefaultVectorComparators.createDefaultComparator(listVector1); + DefaultVectorComparators.createDefaultComparator(listVector1); comparator.attachVectors(listVector1, listVector2); // prefix is smaller @@ -121,11 +118,11 @@ public void testCompareLists() { } try (ListVector listVector1 = createListVector(11); - ListVector listVector2 = createListVector(11)) { + ListVector listVector2 = createListVector(11)) { ((IntVector) listVector2.getDataVector()).set(10, 110); VectorValueComparator comparator = - DefaultVectorComparators.createDefaultComparator(listVector1); + DefaultVectorComparators.createDefaultComparator(listVector1); comparator.attachVectors(listVector1, listVector2); // breaking tie by the last element @@ -133,10 +130,10 @@ public void testCompareLists() { } try (ListVector listVector1 = createListVector(10); - ListVector listVector2 = createListVector(10)) { + ListVector listVector2 = createListVector(10)) { VectorValueComparator comparator = - DefaultVectorComparators.createDefaultComparator(listVector1); + DefaultVectorComparators.createDefaultComparator(listVector1); comparator.attachVectors(listVector1, listVector2); // list vector elements equal @@ -149,9 +146,9 @@ public void testCopiedComparatorForLists() { for (int i = 1; i < 10; i++) { for (int j = 1; j < 10; j++) { try (ListVector listVector1 = createListVector(10); - ListVector listVector2 = createListVector(11)) { + ListVector listVector2 = createListVector(11)) { VectorValueComparator comparator = - DefaultVectorComparators.createDefaultComparator(listVector1); + DefaultVectorComparators.createDefaultComparator(listVector1); comparator.attachVectors(listVector1, listVector2); VectorValueComparator copyComparator = comparator.createNew(); @@ -185,7 +182,7 @@ private FixedSizeListVector createFixedSizeListVector(int count) { @Test public void testCompareFixedSizeLists() { try (FixedSizeListVector listVector1 = createFixedSizeListVector(10); - FixedSizeListVector listVector2 = createFixedSizeListVector(11)) { + FixedSizeListVector listVector2 = createFixedSizeListVector(11)) { VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(listVector1); comparator.attachVectors(listVector1, listVector2); @@ -195,7 +192,7 @@ public void testCompareFixedSizeLists() { } try (FixedSizeListVector listVector1 = createFixedSizeListVector(11); - FixedSizeListVector listVector2 = createFixedSizeListVector(11)) { + FixedSizeListVector listVector2 = createFixedSizeListVector(11)) { ((IntVector) listVector2.getDataVector()).set(10, 110); VectorValueComparator comparator = @@ -207,7 +204,7 @@ public void testCompareFixedSizeLists() { } try (FixedSizeListVector listVector1 = createFixedSizeListVector(10); - FixedSizeListVector listVector2 = createFixedSizeListVector(10)) { + FixedSizeListVector listVector2 = createFixedSizeListVector(10)) { VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(listVector1); @@ -236,7 +233,7 @@ public void testCompareUInt1() { vec.set(9, Byte.MIN_VALUE); VectorValueComparator comparator = - DefaultVectorComparators.createDefaultComparator(vec); + DefaultVectorComparators.createDefaultComparator(vec); comparator.attachVector(vec); assertTrue(comparator.compare(0, 1) < 0); @@ -259,14 +256,21 @@ public void testCompareUInt2() { vec.allocateNew(10); ValueVectorDataPopulator.setVector( - vec, null, (char) (Character.MAX_VALUE - 1), Character.MAX_VALUE, (char) 0, (char) 1, - (char) 2, (char) (Character.MAX_VALUE - 1), null, + vec, + null, + (char) (Character.MAX_VALUE - 1), + Character.MAX_VALUE, + (char) 0, + (char) 1, + (char) 2, + (char) (Character.MAX_VALUE - 1), + null, '\u7FFF', // value for the max 16-byte signed integer '\u8000' // value for the min 16-byte signed integer - ); + ); VectorValueComparator comparator = - DefaultVectorComparators.createDefaultComparator(vec); + DefaultVectorComparators.createDefaultComparator(vec); comparator.attachVector(vec); assertTrue(comparator.compare(0, 1) < 0); @@ -301,7 +305,7 @@ public void testCompareUInt4() { vec.set(9, Integer.MIN_VALUE); VectorValueComparator comparator = - DefaultVectorComparators.createDefaultComparator(vec); + DefaultVectorComparators.createDefaultComparator(vec); comparator.attachVector(vec); assertTrue(comparator.compare(0, 1) < 0); @@ -336,7 +340,7 @@ public void testCompareUInt8() { vec.set(9, Long.MIN_VALUE); VectorValueComparator comparator = - DefaultVectorComparators.createDefaultComparator(vec); + DefaultVectorComparators.createDefaultComparator(vec); comparator.attachVector(vec); assertTrue(comparator.compare(0, 1) < 0); @@ -358,7 +362,16 @@ public void testCompareFloat4() { try (Float4Vector vec = new Float4Vector("", allocator)) { vec.allocateNew(9); ValueVectorDataPopulator.setVector( - vec, -1.1f, 0.0f, 1.0f, null, 1.0f, 2.0f, Float.NaN, Float.NaN, Float.POSITIVE_INFINITY, + vec, + -1.1f, + 0.0f, + 1.0f, + null, + 1.0f, + 2.0f, + Float.NaN, + Float.NaN, + Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY); VectorValueComparator comparator = @@ -393,7 +406,16 @@ public void testCompareFloat8() { try (Float8Vector vec = new Float8Vector("", allocator)) { vec.allocateNew(9); ValueVectorDataPopulator.setVector( - vec, -1.1, 0.0, 1.0, null, 1.0, 2.0, Double.NaN, Double.NaN, Double.POSITIVE_INFINITY, + vec, + -1.1, + 0.0, + 1.0, + null, + 1.0, + 2.0, + Double.NaN, + Double.NaN, + Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY); VectorValueComparator comparator = @@ -488,8 +510,15 @@ public void testCompareShort() { try (SmallIntVector vec = new SmallIntVector("", allocator)) { vec.allocateNew(8); ValueVectorDataPopulator.setVector( - vec, (short) -1, (short) 0, (short) 1, null, (short) 1, (short) 5, - (short) (Short.MIN_VALUE + 1), Short.MAX_VALUE); + vec, + (short) -1, + (short) 0, + (short) 1, + null, + (short) 1, + (short) 5, + (short) (Short.MIN_VALUE + 1), + Short.MAX_VALUE); VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); @@ -519,8 +548,15 @@ public void testCompareByte() { try (TinyIntVector vec = new TinyIntVector("", allocator)) { vec.allocateNew(8); ValueVectorDataPopulator.setVector( - vec, (byte) -1, (byte) 0, (byte) 1, null, (byte) 1, (byte) 5, - (byte) (Byte.MIN_VALUE + 1), Byte.MAX_VALUE); + vec, + (byte) -1, + (byte) 0, + (byte) 1, + null, + (byte) 1, + (byte) 5, + (byte) (Byte.MIN_VALUE + 1), + Byte.MAX_VALUE); VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); @@ -549,8 +585,7 @@ public void testCompareByte() { public void testCompareBit() { try (BitVector vec = new BitVector("", allocator)) { vec.allocateNew(6); - ValueVectorDataPopulator.setVector( - vec, 1, 2, 0, 0, -1, null); + ValueVectorDataPopulator.setVector(vec, 1, 2, 0, 0, -1, null); VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); @@ -691,7 +726,8 @@ public void testCompareDecimal256() { @Test public void testCompareDuration() { try (DurationVector vec = - new DurationVector("", FieldType.nullable(new ArrowType.Duration(TimeUnit.MILLISECOND)), allocator)) { + new DurationVector( + "", FieldType.nullable(new ArrowType.Duration(TimeUnit.MILLISECOND)), allocator)) { vec.allocateNew(8); ValueVectorDataPopulator.setVector( vec, -1L, 0L, 1L, null, 1L, 5L, Long.MIN_VALUE + 1L, Long.MAX_VALUE); @@ -722,7 +758,8 @@ public void testCompareDuration() { @Test public void testCompareIntervalDay() { try (IntervalDayVector vec = - new IntervalDayVector("", FieldType.nullable(new ArrowType.Duration(TimeUnit.MILLISECOND)), allocator)) { + new IntervalDayVector( + "", FieldType.nullable(new ArrowType.Duration(TimeUnit.MILLISECOND)), allocator)) { vec.allocateNew(8); vec.set(0, -1, 0); vec.set(1, 0, 0); @@ -755,8 +792,7 @@ public void testCompareIntervalDay() { @Test public void testCompareTimeMicro() { - try (TimeMicroVector vec = - new TimeMicroVector("", allocator)) { + try (TimeMicroVector vec = new TimeMicroVector("", allocator)) { vec.allocateNew(8); ValueVectorDataPopulator.setVector( vec, -1L, 0L, 1L, null, 1L, 5L, Long.MIN_VALUE + 1L, Long.MAX_VALUE); @@ -816,8 +852,7 @@ public void testCompareTimeMilli() { @Test public void testCompareTimeNano() { - try (TimeNanoVector vec = - new TimeNanoVector("", allocator)) { + try (TimeNanoVector vec = new TimeNanoVector("", allocator)) { vec.allocateNew(8); ValueVectorDataPopulator.setVector( vec, -1L, 0L, 1L, null, 1L, 5L, Long.MIN_VALUE + 1L, Long.MAX_VALUE); @@ -877,8 +912,7 @@ public void testCompareTimeSec() { @Test public void testCompareTimeStamp() { - try (TimeStampMilliVector vec = - new TimeStampMilliVector("", allocator)) { + try (TimeStampMilliVector vec = new TimeStampMilliVector("", allocator)) { vec.allocateNew(8); ValueVectorDataPopulator.setVector( vec, -1L, 0L, 1L, null, 1L, 5L, Long.MIN_VALUE + 1L, Long.MAX_VALUE); @@ -909,7 +943,7 @@ public void testCompareTimeStamp() { @Test public void testCompareFixedSizeBinary() { try (FixedSizeBinaryVector vector1 = new FixedSizeBinaryVector("test1", allocator, 2); - FixedSizeBinaryVector vector2 = new FixedSizeBinaryVector("test1", allocator, 3)) { + FixedSizeBinaryVector vector2 = new FixedSizeBinaryVector("test1", allocator, 3)) { vector1.allocateNew(); vector2.allocateNew(); vector1.set(0, new byte[] {1, 1}); @@ -923,7 +957,7 @@ public void testCompareFixedSizeBinary() { } try (FixedSizeBinaryVector vector1 = new FixedSizeBinaryVector("test1", allocator, 3); - FixedSizeBinaryVector vector2 = new FixedSizeBinaryVector("test1", allocator, 3)) { + FixedSizeBinaryVector vector2 = new FixedSizeBinaryVector("test1", allocator, 3)) { vector1.allocateNew(); vector2.allocateNew(); vector1.set(0, new byte[] {1, 1, 0}); @@ -937,7 +971,7 @@ public void testCompareFixedSizeBinary() { } try (FixedSizeBinaryVector vector1 = new FixedSizeBinaryVector("test1", allocator, 3); - FixedSizeBinaryVector vector2 = new FixedSizeBinaryVector("test1", allocator, 3)) { + FixedSizeBinaryVector vector2 = new FixedSizeBinaryVector("test1", allocator, 3)) { vector1.allocateNew(); vector2.allocateNew(); vector1.set(0, new byte[] {1, 1, 1}); @@ -953,8 +987,8 @@ public void testCompareFixedSizeBinary() { @Test public void testCompareNull() { - try (NullVector vec = new NullVector("test", - FieldType.notNullable(new ArrowType.Int(32, false)))) { + try (NullVector vec = + new NullVector("test", FieldType.notNullable(new ArrowType.Int(32, false)))) { vec.setValueCount(2); VectorValueComparator comparator = @@ -967,12 +1001,14 @@ public void testCompareNull() { @Test public void testCheckNullsOnCompareIsFalseForNonNullableVector() { - try (IntVector vec = new IntVector("not nullable", - FieldType.notNullable(new ArrowType.Int(32, false)), allocator)) { + try (IntVector vec = + new IntVector( + "not nullable", FieldType.notNullable(new ArrowType.Int(32, false)), allocator)) { ValueVectorDataPopulator.setVector(vec, 1, 2, 3, 4); - final VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); + final VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vec); comparator.attachVector(vec); assertFalse(comparator.checkNullsOnCompare()); @@ -981,16 +1017,17 @@ public void testCheckNullsOnCompareIsFalseForNonNullableVector() { @Test public void testCheckNullsOnCompareIsTrueForNullableVector() { - try (IntVector vec = new IntVector("nullable", FieldType.nullable( - new ArrowType.Int(32, false)), allocator); - IntVector vec2 = new IntVector("not-nullable", FieldType.notNullable( - new ArrowType.Int(32, false)), allocator) - ) { + try (IntVector vec = + new IntVector("nullable", FieldType.nullable(new ArrowType.Int(32, false)), allocator); + IntVector vec2 = + new IntVector( + "not-nullable", FieldType.notNullable(new ArrowType.Int(32, false)), allocator)) { ValueVectorDataPopulator.setVector(vec, 1, null, 3, 4); ValueVectorDataPopulator.setVector(vec2, 1, 2, 3, 4); - final VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); + final VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vec); comparator.attachVector(vec); assertTrue(comparator.checkNullsOnCompare()); @@ -1001,17 +1038,18 @@ public void testCheckNullsOnCompareIsTrueForNullableVector() { @Test public void testCheckNullsOnCompareIsFalseWithNoNulls() { - try (IntVector vec = new IntVector("nullable", FieldType.nullable( - new ArrowType.Int(32, false)), allocator); - IntVector vec2 = new IntVector("also-nullable", FieldType.nullable( - new ArrowType.Int(32, false)), allocator) - ) { + try (IntVector vec = + new IntVector("nullable", FieldType.nullable(new ArrowType.Int(32, false)), allocator); + IntVector vec2 = + new IntVector( + "also-nullable", FieldType.nullable(new ArrowType.Int(32, false)), allocator)) { // no null values ValueVectorDataPopulator.setVector(vec, 1, 2, 3, 4); ValueVectorDataPopulator.setVector(vec2, 1, 2, 3, 4); - final VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); + final VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vec); comparator.attachVector(vec); assertFalse(comparator.checkNullsOnCompare()); @@ -1022,13 +1060,14 @@ public void testCheckNullsOnCompareIsFalseWithNoNulls() { @Test public void testCheckNullsOnCompareIsTrueWithEmptyVectors() { - try (IntVector vec = new IntVector("nullable", FieldType.nullable( - new ArrowType.Int(32, false)), allocator); - IntVector vec2 = new IntVector("also-nullable", FieldType.nullable( - new ArrowType.Int(32, false)), allocator) - ) { + try (IntVector vec = + new IntVector("nullable", FieldType.nullable(new ArrowType.Int(32, false)), allocator); + IntVector vec2 = + new IntVector( + "also-nullable", FieldType.nullable(new ArrowType.Int(32, false)), allocator)) { - final VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); + final VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vec); comparator.attachVector(vec2); assertTrue(comparator.checkNullsOnCompare()); diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestFixedWidthInPlaceVectorSorter.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestFixedWidthInPlaceVectorSorter.java index 91ef52017df4d..ed5aadfcda04c 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestFixedWidthInPlaceVectorSorter.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestFixedWidthInPlaceVectorSorter.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.sort; import static org.junit.Assert.assertEquals; @@ -22,7 +21,6 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import java.util.stream.IntStream; - import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.IntVector; @@ -32,9 +30,7 @@ import org.junit.Before; import org.junit.Test; -/** - * Test cases for {@link FixedWidthInPlaceVectorSorter}. - */ +/** Test cases for {@link FixedWidthInPlaceVectorSorter}. */ public class TestFixedWidthInPlaceVectorSorter { private BufferAllocator allocator; @@ -69,7 +65,8 @@ public void testSortInt() { // sort the vector FixedWidthInPlaceVectorSorter sorter = new FixedWidthInPlaceVectorSorter(); - VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vec); sorter.sortInPlace(vec, comparator); @@ -90,8 +87,8 @@ public void testSortInt() { } /** - * Tests the worst case for quick sort. - * It may cause stack overflow if the algorithm is implemented as a recursive algorithm. + * Tests the worst case for quick sort. It may cause stack overflow if the algorithm is + * implemented as a recursive algorithm. */ @Test public void testSortLargeIncreasingInt() { @@ -107,7 +104,8 @@ public void testSortLargeIncreasingInt() { // sort the vector FixedWidthInPlaceVectorSorter sorter = new FixedWidthInPlaceVectorSorter(); - VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vec); sorter.sortInPlace(vec, comparator); @@ -133,7 +131,8 @@ public void testChoosePivot() { vec.setValueCount(vectorLength); FixedWidthInPlaceVectorSorter sorter = new FixedWidthInPlaceVectorSorter(); - VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vec); try (IntVector pivotBuffer = (IntVector) vec.getField().createVector(allocator)) { // setup internal data structures @@ -164,16 +163,15 @@ public void testChoosePivot() { } } - /** - * Evaluates choosing pivot for all possible permutations of 3 numbers. - */ + /** Evaluates choosing pivot for all possible permutations of 3 numbers. */ @Test public void testChoosePivotAllPermutes() { try (IntVector vec = new IntVector("", allocator)) { vec.allocateNew(3); FixedWidthInPlaceVectorSorter sorter = new FixedWidthInPlaceVectorSorter(); - VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vec); try (IntVector pivotBuffer = (IntVector) vec.getField().createVector(allocator)) { // setup internal data structures @@ -216,25 +214,25 @@ public void testChoosePivotAllPermutes() { @Test public void testSortInt2() { try (IntVector vector = new IntVector("vector", allocator)) { - ValueVectorDataPopulator.setVector(vector, - 0, 1, 2, 3, 4, 5, 30, 31, 32, 33, - 34, 35, 60, 61, 62, 63, 64, 65, 6, 7, - 8, 9, 10, 11, 36, 37, 38, 39, 40, 41, - 66, 67, 68, 69, 70, 71); + ValueVectorDataPopulator.setVector( + vector, 0, 1, 2, 3, 4, 5, 30, 31, 32, 33, 34, 35, 60, 61, 62, 63, 64, 65, 6, 7, 8, 9, 10, + 11, 36, 37, 38, 39, 40, 41, 66, 67, 68, 69, 70, 71); FixedWidthInPlaceVectorSorter sorter = new FixedWidthInPlaceVectorSorter(); - VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vector); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vector); sorter.sortInPlace(vector, comparator); int[] actual = new int[vector.getValueCount()]; - IntStream.range(0, vector.getValueCount()).forEach( - i -> actual[i] = vector.get(i)); + IntStream.range(0, vector.getValueCount()).forEach(i -> actual[i] = vector.get(i)); assertArrayEquals( - new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, - 11, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, - 40, 41, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71}, actual); + new int[] { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71 + }, + actual); } } } diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestFixedWidthOutOfPlaceVectorSorter.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestFixedWidthOutOfPlaceVectorSorter.java index cc13e7f8ceaee..4096897c20a05 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestFixedWidthOutOfPlaceVectorSorter.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestFixedWidthOutOfPlaceVectorSorter.java @@ -14,14 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.sort; import static org.junit.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import java.util.stream.IntStream; - import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BaseFixedWidthVector; @@ -37,9 +35,7 @@ import org.junit.Before; import org.junit.Test; -/** - * Test cases for {@link FixedWidthOutOfPlaceVectorSorter}. - */ +/** Test cases for {@link FixedWidthOutOfPlaceVectorSorter}. */ public class TestFixedWidthOutOfPlaceVectorSorter extends TestOutOfPlaceVectorSorter { private BufferAllocator allocator; @@ -49,7 +45,9 @@ public TestFixedWidthOutOfPlaceVectorSorter(boolean generalSorter) { } OutOfPlaceVectorSorter getSorter() { - return generalSorter ? new GeneralOutOfPlaceVectorSorter<>() : new FixedWidthOutOfPlaceVectorSorter<>(); + return generalSorter + ? new GeneralOutOfPlaceVectorSorter<>() + : new FixedWidthOutOfPlaceVectorSorter<>(); } @Before @@ -82,10 +80,11 @@ public void testSortByte() { // sort the vector OutOfPlaceVectorSorter sorter = getSorter(); - VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vec); TinyIntVector sortedVec = - (TinyIntVector) vec.getField().getFieldType().createNewSingleVector("", allocator, null); + (TinyIntVector) vec.getField().getFieldType().createNewSingleVector("", allocator, null); sortedVec.allocateNew(vec.getValueCount()); sortedVec.setValueCount(vec.getValueCount()); @@ -129,10 +128,11 @@ public void testSortShort() { // sort the vector OutOfPlaceVectorSorter sorter = getSorter(); - VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vec); SmallIntVector sortedVec = - (SmallIntVector) vec.getField().getFieldType().createNewSingleVector("", allocator, null); + (SmallIntVector) vec.getField().getFieldType().createNewSingleVector("", allocator, null); sortedVec.allocateNew(vec.getValueCount()); sortedVec.setValueCount(vec.getValueCount()); @@ -176,9 +176,11 @@ public void testSortInt() { // sort the vector OutOfPlaceVectorSorter sorter = getSorter(); - VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vec); - IntVector sortedVec = (IntVector) vec.getField().getFieldType().createNewSingleVector("", allocator, null); + IntVector sortedVec = + (IntVector) vec.getField().getFieldType().createNewSingleVector("", allocator, null); sortedVec.allocateNew(vec.getValueCount()); sortedVec.setValueCount(vec.getValueCount()); @@ -222,9 +224,11 @@ public void testSortLong() { // sort the vector OutOfPlaceVectorSorter sorter = getSorter(); - VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vec); - BigIntVector sortedVec = (BigIntVector) vec.getField().getFieldType().createNewSingleVector("", allocator, null); + BigIntVector sortedVec = + (BigIntVector) vec.getField().getFieldType().createNewSingleVector("", allocator, null); sortedVec.allocateNew(vec.getValueCount()); sortedVec.setValueCount(vec.getValueCount()); @@ -268,9 +272,11 @@ public void testSortFloat() { // sort the vector OutOfPlaceVectorSorter sorter = getSorter(); - VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vec); - Float4Vector sortedVec = (Float4Vector) vec.getField().getFieldType().createNewSingleVector("", allocator, null); + Float4Vector sortedVec = + (Float4Vector) vec.getField().getFieldType().createNewSingleVector("", allocator, null); sortedVec.allocateNew(vec.getValueCount()); sortedVec.setValueCount(vec.getValueCount()); @@ -314,9 +320,11 @@ public void testSortDouble() { // sort the vector OutOfPlaceVectorSorter sorter = getSorter(); - VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vec); - Float8Vector sortedVec = (Float8Vector) vec.getField().getFieldType().createNewSingleVector("", allocator, null); + Float8Vector sortedVec = + (Float8Vector) vec.getField().getFieldType().createNewSingleVector("", allocator, null); sortedVec.allocateNew(vec.getValueCount()); sortedVec.setValueCount(vec.getValueCount()); @@ -343,17 +351,17 @@ public void testSortDouble() { @Test public void testSortInt2() { try (IntVector vec = new IntVector("", allocator)) { - ValueVectorDataPopulator.setVector(vec, - 0, 1, 2, 3, 4, 5, 30, 31, 32, 33, - 34, 35, 60, 61, 62, 63, 64, 65, 6, 7, - 8, 9, 10, 11, 36, 37, 38, 39, 40, 41, - 66, 67, 68, 69, 70, 71); + ValueVectorDataPopulator.setVector( + vec, 0, 1, 2, 3, 4, 5, 30, 31, 32, 33, 34, 35, 60, 61, 62, 63, 64, 65, 6, 7, 8, 9, 10, 11, + 36, 37, 38, 39, 40, 41, 66, 67, 68, 69, 70, 71); // sort the vector OutOfPlaceVectorSorter sorter = getSorter(); - VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vec); - try (IntVector sortedVec = (IntVector) vec.getField().getFieldType().createNewSingleVector("", allocator, null)) { + try (IntVector sortedVec = + (IntVector) vec.getField().getFieldType().createNewSingleVector("", allocator, null)) { sortedVec.allocateNew(vec.getValueCount()); sortedVec.setValueCount(vec.getValueCount()); @@ -361,13 +369,14 @@ public void testSortInt2() { // verify results int[] actual = new int[sortedVec.getValueCount()]; - IntStream.range(0, sortedVec.getValueCount()).forEach( - i -> actual[i] = sortedVec.get(i)); + IntStream.range(0, sortedVec.getValueCount()).forEach(i -> actual[i] = sortedVec.get(i)); assertArrayEquals( - new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, - 11, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, - 40, 41, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71}, actual); + new int[] { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71 + }, + actual); } } } diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestFixedWidthSorting.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestFixedWidthSorting.java index 80c72b4e21a27..a92cc77818f4a 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestFixedWidthSorting.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestFixedWidthSorting.java @@ -14,14 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.sort; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.function.Function; - import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BaseFixedWidthVector; @@ -37,9 +35,7 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -/** - * Test sorting fixed width vectors with random data. - */ +/** Test sorting fixed width vectors with random data. */ @RunWith(Parameterized.class) public class TestFixedWidthSorting> { @@ -70,8 +66,12 @@ public void shutdown() { } public TestFixedWidthSorting( - int length, double nullFraction, boolean inPlace, String desc, - Function vectorGenerator, TestSortingUtil.DataGenerator dataGenerator) { + int length, + double nullFraction, + boolean inPlace, + String desc, + Function vectorGenerator, + TestSortingUtil.DataGenerator dataGenerator) { this.length = length; this.nullFraction = nullFraction; this.inPlace = inPlace; @@ -94,7 +94,8 @@ void sortInPlace() { TestSortingUtil.sortArray(array); FixedWidthInPlaceVectorSorter sorter = new FixedWidthInPlaceVectorSorter(); - VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vector); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vector); sorter.sortInPlace(vector, comparator); @@ -109,9 +110,11 @@ void sortOutOfPlace() { // sort the vector FixedWidthOutOfPlaceVectorSorter sorter = new FixedWidthOutOfPlaceVectorSorter(); - VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vector); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vector); - try (V sortedVec = (V) vector.getField().getFieldType().createNewSingleVector("", allocator, null)) { + try (V sortedVec = + (V) vector.getField().getFieldType().createNewSingleVector("", allocator, null)) { sortedVec.allocateNew(vector.getValueCount()); sortedVec.setValueCount(vector.getValueCount()); @@ -123,47 +126,78 @@ void sortOutOfPlace() { } } - @Parameterized.Parameters(name = "length = {0}, null fraction = {1}, in place = {2}, vector = {3}") + @Parameterized.Parameters( + name = "length = {0}, null fraction = {1}, in place = {2}, vector = {3}") public static Collection getParameters() { List params = new ArrayList<>(); for (int length : VECTOR_LENGTHS) { for (double nullFrac : NULL_FRACTIONS) { for (boolean inPlace : new boolean[] {true, false}) { - params.add(new Object[] { - length, nullFrac, inPlace, "TinyIntVector", - (Function) allocator -> new TinyIntVector("vector", allocator), - TestSortingUtil.TINY_INT_GENERATOR - }); - - params.add(new Object[] { - length, nullFrac, inPlace, "SmallIntVector", - (Function) allocator -> new SmallIntVector("vector", allocator), - TestSortingUtil.SMALL_INT_GENERATOR - }); - - params.add(new Object[] { - length, nullFrac, inPlace, "IntVector", - (Function) allocator -> new IntVector("vector", allocator), - TestSortingUtil.INT_GENERATOR - }); - - params.add(new Object[] { - length, nullFrac, inPlace, "BigIntVector", - (Function) allocator -> new BigIntVector("vector", allocator), - TestSortingUtil.LONG_GENERATOR - }); - - params.add(new Object[] { - length, nullFrac, inPlace, "Float4Vector", - (Function) allocator -> new Float4Vector("vector", allocator), - TestSortingUtil.FLOAT_GENERATOR - }); - - params.add(new Object[] { - length, nullFrac, inPlace, "Float8Vector", - (Function) allocator -> new Float8Vector("vector", allocator), - TestSortingUtil.DOUBLE_GENERATOR - }); + params.add( + new Object[] { + length, + nullFrac, + inPlace, + "TinyIntVector", + (Function) + allocator -> new TinyIntVector("vector", allocator), + TestSortingUtil.TINY_INT_GENERATOR + }); + + params.add( + new Object[] { + length, + nullFrac, + inPlace, + "SmallIntVector", + (Function) + allocator -> new SmallIntVector("vector", allocator), + TestSortingUtil.SMALL_INT_GENERATOR + }); + + params.add( + new Object[] { + length, + nullFrac, + inPlace, + "IntVector", + (Function) + allocator -> new IntVector("vector", allocator), + TestSortingUtil.INT_GENERATOR + }); + + params.add( + new Object[] { + length, + nullFrac, + inPlace, + "BigIntVector", + (Function) + allocator -> new BigIntVector("vector", allocator), + TestSortingUtil.LONG_GENERATOR + }); + + params.add( + new Object[] { + length, + nullFrac, + inPlace, + "Float4Vector", + (Function) + allocator -> new Float4Vector("vector", allocator), + TestSortingUtil.FLOAT_GENERATOR + }); + + params.add( + new Object[] { + length, + nullFrac, + inPlace, + "Float8Vector", + (Function) + allocator -> new Float8Vector("vector", allocator), + TestSortingUtil.DOUBLE_GENERATOR + }); } } } diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestGeneralOutOfPlaceVectorSorter.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestGeneralOutOfPlaceVectorSorter.java index 07a6b545ddaa2..9e796a98ab790 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestGeneralOutOfPlaceVectorSorter.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestGeneralOutOfPlaceVectorSorter.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.sort; import static org.junit.Assert.assertEquals; @@ -30,9 +29,7 @@ import org.junit.Before; import org.junit.Test; -/** - * Test cases for {@link GeneralOutOfPlaceVectorSorter}. - */ +/** Test cases for {@link GeneralOutOfPlaceVectorSorter}. */ public class TestGeneralOutOfPlaceVectorSorter { private BufferAllocator allocator; @@ -49,30 +46,33 @@ public void shutdown() { VectorValueComparator getComparator(StructVector structVector) { IntVector child0 = structVector.getChild("column0", IntVector.class); - VectorValueComparator childComp0 = DefaultVectorComparators.createDefaultComparator(child0); + VectorValueComparator childComp0 = + DefaultVectorComparators.createDefaultComparator(child0); childComp0.attachVector(child0); IntVector child1 = structVector.getChild("column1", IntVector.class); - VectorValueComparator childComp1 = DefaultVectorComparators.createDefaultComparator(child1); + VectorValueComparator childComp1 = + DefaultVectorComparators.createDefaultComparator(child1); childComp1.attachVector(child1); - VectorValueComparator comp = new VectorValueComparator() { - - @Override - public int compareNotNull(int index1, int index2) { - // compare values by lexicographic order - int result0 = childComp0.compare(index1, index2); - if (result0 != 0) { - return result0; - } - return childComp1.compare(index1, index2); - } - - @Override - public VectorValueComparator createNew() { - return this; - } - }; + VectorValueComparator comp = + new VectorValueComparator() { + + @Override + public int compareNotNull(int index1, int index2) { + // compare values by lexicographic order + int result0 = childComp0.compare(index1, index2); + if (result0 != 0) { + return result0; + } + return childComp1.compare(index1, index2); + } + + @Override + public VectorValueComparator createNew() { + return this; + } + }; return comp; } @@ -81,17 +81,21 @@ public VectorValueComparator createNew() { public void testSortStructVector() { final int vectorLength = 7; try (StructVector srcVector = StructVector.empty("src struct", allocator); - StructVector dstVector = StructVector.empty("dst struct", allocator)) { + StructVector dstVector = StructVector.empty("dst struct", allocator)) { IntVector srcChild0 = - srcVector.addOrGet("column0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); + srcVector.addOrGet( + "column0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); IntVector srcChild1 = - srcVector.addOrGet("column1", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); + srcVector.addOrGet( + "column1", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); IntVector dstChild0 = - dstVector.addOrGet("column0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); + dstVector.addOrGet( + "column0", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); IntVector dstChild1 = - dstVector.addOrGet("column1", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); + dstVector.addOrGet( + "column1", FieldType.nullable(new ArrowType.Int(32, true)), IntVector.class); // src struct vector values: // [ @@ -128,15 +132,16 @@ public void testSortStructVector() { // validate results assertEquals(vectorLength, dstVector.getValueCount()); assertEquals( - "[" + - "null, " + - "{\"column1\":3}, " + - "{\"column0\":2,\"column1\":1}, " + - "{\"column0\":3,\"column1\":4}, " + - "{\"column0\":5,\"column1\":4}, " + - "{\"column0\":6,\"column1\":6}, " + - "{\"column0\":7}" + - "]", dstVector.toString()); + "[" + + "null, " + + "{\"column1\":3}, " + + "{\"column0\":2,\"column1\":1}, " + + "{\"column0\":3,\"column1\":4}, " + + "{\"column0\":5,\"column1\":4}, " + + "{\"column0\":6,\"column1\":6}, " + + "{\"column0\":7}" + + "]", + dstVector.toString()); } } } diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestIndexSorter.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestIndexSorter.java index 99e22f8bdcd5c..bc8aac08b61e4 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestIndexSorter.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestIndexSorter.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.sort; import static org.junit.Assert.assertEquals; @@ -28,9 +27,7 @@ import org.junit.Before; import org.junit.Test; -/** - * Test cases for {@link IndexSorter}. - */ +/** Test cases for {@link IndexSorter}. */ public class TestIndexSorter { private BufferAllocator allocator; @@ -56,14 +53,15 @@ public void testIndexSort() { // sort the index IndexSorter indexSorter = new IndexSorter<>(); - DefaultVectorComparators.IntComparator intComparator = new DefaultVectorComparators.IntComparator(); + DefaultVectorComparators.IntComparator intComparator = + new DefaultVectorComparators.IntComparator(); intComparator.attachVector(vec); IntVector indices = new IntVector("", allocator); indices.setValueCount(10); indexSorter.sort(vec, indices, intComparator); - int[] expected = new int[]{6, 9, 1, 3, 0, 4, 5, 7, 2, 8}; + int[] expected = new int[] {6, 9, 1, 3, 0, 4, 5, 7, 2, 8}; for (int i = 0; i < expected.length; i++) { assertTrue(!indices.isNull(i)); @@ -74,8 +72,8 @@ public void testIndexSort() { } /** - * Tests the worst case for quick sort. - * It may cause stack overflow if the algorithm is implemented as a recursive algorithm. + * Tests the worst case for quick sort. It may cause stack overflow if the algorithm is + * implemented as a recursive algorithm. */ @Test public void testSortLargeIncreasingInt() { @@ -91,7 +89,8 @@ public void testSortLargeIncreasingInt() { // sort the vector IndexSorter indexSorter = new IndexSorter<>(); - DefaultVectorComparators.IntComparator intComparator = new DefaultVectorComparators.IntComparator(); + DefaultVectorComparators.IntComparator intComparator = + new DefaultVectorComparators.IntComparator(); intComparator.attachVector(vec); try (IntVector indices = new IntVector("", allocator)) { @@ -110,7 +109,7 @@ public void testSortLargeIncreasingInt() { public void testChoosePivot() { final int vectorLength = 100; try (IntVector vec = new IntVector("vector", allocator); - IntVector indices = new IntVector("indices", allocator)) { + IntVector indices = new IntVector("indices", allocator)) { vec.allocateNew(vectorLength); indices.allocateNew(vectorLength); @@ -122,7 +121,8 @@ public void testChoosePivot() { vec.setValueCount(vectorLength); indices.setValueCount(vectorLength); - VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vec); // setup internal data structures comparator.attachVector(vec); @@ -147,17 +147,16 @@ public void testChoosePivot() { } } - /** - * Evaluates choosing pivot for all possible permutations of 3 numbers. - */ + /** Evaluates choosing pivot for all possible permutations of 3 numbers. */ @Test public void testChoosePivotAllPermutes() { try (IntVector vec = new IntVector("vector", allocator); - IntVector indices = new IntVector("indices", allocator)) { + IntVector indices = new IntVector("indices", allocator)) { vec.allocateNew(); indices.allocateNew(); - VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vec); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vec); // setup internal data structures comparator.attachVector(vec); diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestInsertionSorter.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestInsertionSorter.java index ba9c42913c0d9..3b16ac30d4ff4 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestInsertionSorter.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestInsertionSorter.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.sort; import static org.junit.Assert.assertFalse; @@ -28,9 +27,7 @@ import org.junit.Before; import org.junit.Test; -/** - * Test cases for {@link InsertionSorter}. - */ +/** Test cases for {@link InsertionSorter}. */ public class TestInsertionSorter { private BufferAllocator allocator; @@ -49,7 +46,7 @@ public void shutdown() { private void testSortIntVectorRange(int start, int end, int[] expected) { try (IntVector vector = new IntVector("vector", allocator); - IntVector buffer = new IntVector("buffer", allocator)) { + IntVector buffer = new IntVector("buffer", allocator)) { buffer.allocateNew(1); @@ -81,7 +78,7 @@ public void testSortIntVector() { private void testSortIndicesRange(int start, int end, int[] expectedIndices) { try (IntVector vector = new IntVector("vector", allocator); - IntVector indices = new IntVector("indices", allocator)) { + IntVector indices = new IntVector("indices", allocator)) { ValueVectorDataPopulator.setVector(vector, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); ValueVectorDataPopulator.setVector(indices, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestOffHeapIntStack.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestOffHeapIntStack.java index 321ca226d7e1d..025576f08e248 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestOffHeapIntStack.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestOffHeapIntStack.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.sort; import static junit.framework.TestCase.assertEquals; @@ -26,9 +25,7 @@ import org.junit.Before; import org.junit.Test; -/** - * Test cases for {@link OffHeapIntStack}. - */ +/** Test cases for {@link OffHeapIntStack}. */ public class TestOffHeapIntStack { private BufferAllocator allocator; diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestOutOfPlaceVectorSorter.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestOutOfPlaceVectorSorter.java index 66b75cbccac3e..4f6a8489c43ea 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestOutOfPlaceVectorSorter.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestOutOfPlaceVectorSorter.java @@ -14,19 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.sort; import java.util.ArrayList; import java.util.Collection; import java.util.List; - import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -/** - * Test cases for out-of-place sorters. - */ +/** Test cases for out-of-place sorters. */ @RunWith(Parameterized.class) public abstract class TestOutOfPlaceVectorSorter { diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestSortingUtil.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestSortingUtil.java index e22b22d4e6757..24b2c752d0863 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestSortingUtil.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestSortingUtil.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.sort; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -25,7 +24,6 @@ import java.util.Random; import java.util.function.BiConsumer; import java.util.function.Supplier; - import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.Float4Vector; import org.apache.arrow.vector.Float8Vector; @@ -37,50 +35,59 @@ import org.apache.arrow.vector.testing.RandomDataGenerator; import org.apache.arrow.vector.testing.ValueVectorDataPopulator; -/** - * Utilities for sorting related utilities. - */ +/** Utilities for sorting related utilities. */ public class TestSortingUtil { static final Random random = new Random(0); - static final DataGenerator TINY_INT_GENERATOR = new DataGenerator<>( - RandomDataGenerator.TINY_INT_GENERATOR, - (vector, array) -> ValueVectorDataPopulator.setVector(vector, array), Byte.class); - - static final DataGenerator SMALL_INT_GENERATOR = new DataGenerator<>( - RandomDataGenerator.SMALL_INT_GENERATOR, - (vector, array) -> ValueVectorDataPopulator.setVector(vector, array), Short.class); - - static final DataGenerator INT_GENERATOR = new DataGenerator<>( - RandomDataGenerator.INT_GENERATOR, - (vector, array) -> ValueVectorDataPopulator.setVector(vector, array), Integer.class); - - static final DataGenerator LONG_GENERATOR = new DataGenerator<>( - RandomDataGenerator.LONG_GENERATOR, - (vector, array) -> ValueVectorDataPopulator.setVector(vector, array), Long.class); - - static final DataGenerator FLOAT_GENERATOR = new DataGenerator<>( - RandomDataGenerator.FLOAT_GENERATOR, - (vector, array) -> ValueVectorDataPopulator.setVector(vector, array), Float.class); - - static final DataGenerator DOUBLE_GENERATOR = new DataGenerator<>( - RandomDataGenerator.DOUBLE_GENERATOR, - (vector, array) -> ValueVectorDataPopulator.setVector(vector, array), Double.class); - - static final DataGenerator STRING_GENERATOR = new DataGenerator<>( - () -> { - int strLength = random.nextInt(20) + 1; - return generateRandomString(strLength); - }, - (vector, array) -> ValueVectorDataPopulator.setVector(vector, array), String.class); - - private TestSortingUtil() { - } - - /** - * Verify that a vector is equal to an array. - */ + static final DataGenerator TINY_INT_GENERATOR = + new DataGenerator<>( + RandomDataGenerator.TINY_INT_GENERATOR, + (vector, array) -> ValueVectorDataPopulator.setVector(vector, array), + Byte.class); + + static final DataGenerator SMALL_INT_GENERATOR = + new DataGenerator<>( + RandomDataGenerator.SMALL_INT_GENERATOR, + (vector, array) -> ValueVectorDataPopulator.setVector(vector, array), + Short.class); + + static final DataGenerator INT_GENERATOR = + new DataGenerator<>( + RandomDataGenerator.INT_GENERATOR, + (vector, array) -> ValueVectorDataPopulator.setVector(vector, array), + Integer.class); + + static final DataGenerator LONG_GENERATOR = + new DataGenerator<>( + RandomDataGenerator.LONG_GENERATOR, + (vector, array) -> ValueVectorDataPopulator.setVector(vector, array), + Long.class); + + static final DataGenerator FLOAT_GENERATOR = + new DataGenerator<>( + RandomDataGenerator.FLOAT_GENERATOR, + (vector, array) -> ValueVectorDataPopulator.setVector(vector, array), + Float.class); + + static final DataGenerator DOUBLE_GENERATOR = + new DataGenerator<>( + RandomDataGenerator.DOUBLE_GENERATOR, + (vector, array) -> ValueVectorDataPopulator.setVector(vector, array), + Double.class); + + static final DataGenerator STRING_GENERATOR = + new DataGenerator<>( + () -> { + int strLength = random.nextInt(20) + 1; + return generateRandomString(strLength); + }, + (vector, array) -> ValueVectorDataPopulator.setVector(vector, array), + String.class); + + private TestSortingUtil() {} + + /** Verify that a vector is equal to an array. */ public static void verifyResults(V vector, U[] expected) { assertEquals(vector.getValueCount(), expected.length); for (int i = 0; i < expected.length; i++) { @@ -88,30 +95,28 @@ public static void verifyResults(V vector, U[] expect } } - /** - * Sort an array with null values come first. - */ + /** Sort an array with null values come first. */ public static > void sortArray(U[] array) { - Arrays.sort(array, (a, b) -> { - if (a == null || b == null) { - if (a == null && b == null) { - return 0; - } - - // exactly one is null - if (a == null) { - return -1; - } else { - return 1; - } - } - return a.compareTo(b); - }); + Arrays.sort( + array, + (a, b) -> { + if (a == null || b == null) { + if (a == null && b == null) { + return 0; + } + + // exactly one is null + if (a == null) { + return -1; + } else { + return 1; + } + } + return a.compareTo(b); + }); } - /** - * Generate a string with alphabetic characters only. - */ + /** Generate a string with alphabetic characters only. */ static String generateRandomString(int length) { byte[] str = new byte[length]; final int lower = 'a'; @@ -128,6 +133,7 @@ static String generateRandomString(int length) { /** * Utility to generate data for testing. + * * @param vector type. * @param data element type. */ @@ -139,8 +145,7 @@ static class DataGenerator> { final Class clazz; - DataGenerator( - Supplier dataGenerator, BiConsumer vectorPopulator, Class clazz) { + DataGenerator(Supplier dataGenerator, BiConsumer vectorPopulator, Class clazz) { this.dataGenerator = dataGenerator; this.vectorPopulator = vectorPopulator; this.clazz = clazz; @@ -148,6 +153,7 @@ static class DataGenerator> { /** * Populate the vector according to the specified parameters. + * * @param vector the vector to populate. * @param length vector length. * @param nullFraction the fraction of null values. diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestStableVectorComparator.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestStableVectorComparator.java index f2de5d23fce89..ce15940c1df3d 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestStableVectorComparator.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestStableVectorComparator.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.sort; import static org.junit.Assert.assertEquals; @@ -22,7 +21,6 @@ import java.nio.charset.StandardCharsets; import java.util.Objects; - import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.VarCharVector; @@ -31,9 +29,7 @@ import org.junit.Test; import org.junit.jupiter.api.Assertions; -/** - * Test cases for {@link StableVectorComparator}. - */ +/** Test cases for {@link StableVectorComparator}. */ public class TestStableVectorComparator { private BufferAllocator allocator; @@ -62,7 +58,8 @@ public void testCompare() { vec.set(4, "a".getBytes(StandardCharsets.UTF_8)); VectorValueComparator comparator = new TestVarCharSorter(); - VectorValueComparator stableComparator = new StableVectorComparator<>(comparator); + VectorValueComparator stableComparator = + new StableVectorComparator<>(comparator); stableComparator.attachVector(vec); assertTrue(stableComparator.compare(0, 1) > 0); @@ -95,10 +92,12 @@ public void testStableSortString() { // sort the vector VariableWidthOutOfPlaceVectorSorter sorter = new VariableWidthOutOfPlaceVectorSorter(); VectorValueComparator comparator = new TestVarCharSorter(); - VectorValueComparator stableComparator = new StableVectorComparator<>(comparator); + VectorValueComparator stableComparator = + new StableVectorComparator<>(comparator); try (VarCharVector sortedVec = - (VarCharVector) vec.getField().getFieldType().createNewSingleVector("", allocator, null)) { + (VarCharVector) + vec.getField().getFieldType().createNewSingleVector("", allocator, null)) { sortedVec.allocateNew(vec.getByteCapacity(), vec.getValueCount()); sortedVec.setLastSet(vec.getValueCount() - 1); sortedVec.setValueCount(vec.getValueCount()); @@ -107,23 +106,32 @@ public void testStableSortString() { // verify results // the results are stable - assertEquals("0", new String(Objects.requireNonNull(sortedVec.get(0)), StandardCharsets.UTF_8)); - assertEquals("01", new String(Objects.requireNonNull(sortedVec.get(1)), StandardCharsets.UTF_8)); - assertEquals("0c", new String(Objects.requireNonNull(sortedVec.get(2)), StandardCharsets.UTF_8)); - assertEquals("a", new String(Objects.requireNonNull(sortedVec.get(3)), StandardCharsets.UTF_8)); - assertEquals("abc", new String(Objects.requireNonNull(sortedVec.get(4)), StandardCharsets.UTF_8)); - assertEquals("aa", new String(Objects.requireNonNull(sortedVec.get(5)), StandardCharsets.UTF_8)); - assertEquals("a1", new String(Objects.requireNonNull(sortedVec.get(6)), StandardCharsets.UTF_8)); - assertEquals("abcdefg", new String(Objects.requireNonNull(sortedVec.get(7)), StandardCharsets.UTF_8)); - assertEquals("accc", new String(Objects.requireNonNull(sortedVec.get(8)), StandardCharsets.UTF_8)); - assertEquals("afds", new String(Objects.requireNonNull(sortedVec.get(9)), StandardCharsets.UTF_8)); + assertEquals( + "0", new String(Objects.requireNonNull(sortedVec.get(0)), StandardCharsets.UTF_8)); + assertEquals( + "01", new String(Objects.requireNonNull(sortedVec.get(1)), StandardCharsets.UTF_8)); + assertEquals( + "0c", new String(Objects.requireNonNull(sortedVec.get(2)), StandardCharsets.UTF_8)); + assertEquals( + "a", new String(Objects.requireNonNull(sortedVec.get(3)), StandardCharsets.UTF_8)); + assertEquals( + "abc", new String(Objects.requireNonNull(sortedVec.get(4)), StandardCharsets.UTF_8)); + assertEquals( + "aa", new String(Objects.requireNonNull(sortedVec.get(5)), StandardCharsets.UTF_8)); + assertEquals( + "a1", new String(Objects.requireNonNull(sortedVec.get(6)), StandardCharsets.UTF_8)); + assertEquals( + "abcdefg", + new String(Objects.requireNonNull(sortedVec.get(7)), StandardCharsets.UTF_8)); + assertEquals( + "accc", new String(Objects.requireNonNull(sortedVec.get(8)), StandardCharsets.UTF_8)); + assertEquals( + "afds", new String(Objects.requireNonNull(sortedVec.get(9)), StandardCharsets.UTF_8)); } } } - /** - * Utility comparator that compares varchars by the first character. - */ + /** Utility comparator that compares varchars by the first character. */ private static class TestVarCharSorter extends VectorValueComparator { @Override diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestVariableWidthOutOfPlaceVectorSorter.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestVariableWidthOutOfPlaceVectorSorter.java index 2486034f1fa32..b3f2539fa53c2 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestVariableWidthOutOfPlaceVectorSorter.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestVariableWidthOutOfPlaceVectorSorter.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.sort; import static org.junit.Assert.assertEquals; @@ -22,7 +21,6 @@ import java.nio.charset.StandardCharsets; import java.util.Objects; - import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BaseVariableWidthVector; @@ -32,9 +30,7 @@ import org.junit.Before; import org.junit.Test; -/** - * Test cases for {@link VariableWidthOutOfPlaceVectorSorter}. - */ +/** Test cases for {@link VariableWidthOutOfPlaceVectorSorter}. */ public class TestVariableWidthOutOfPlaceVectorSorter extends TestOutOfPlaceVectorSorter { private BufferAllocator allocator; @@ -44,10 +40,11 @@ public TestVariableWidthOutOfPlaceVectorSorter(boolean generalSorter) { } OutOfPlaceVectorSorter getSorter() { - return generalSorter ? new GeneralOutOfPlaceVectorSorter<>() : new VariableWidthOutOfPlaceVectorSorter(); + return generalSorter + ? new GeneralOutOfPlaceVectorSorter<>() + : new VariableWidthOutOfPlaceVectorSorter(); } - @Before public void prepare() { allocator = new RootAllocator(1024 * 1024); @@ -79,10 +76,10 @@ public void testSortString() { // sort the vector OutOfPlaceVectorSorter sorter = getSorter(); VectorValueComparator comparator = - DefaultVectorComparators.createDefaultComparator(vec); + DefaultVectorComparators.createDefaultComparator(vec); VarCharVector sortedVec = - (VarCharVector) vec.getField().getFieldType().createNewSingleVector("", allocator, null); + (VarCharVector) vec.getField().getFieldType().createNewSingleVector("", allocator, null); sortedVec.allocateNew(vec.getByteCapacity(), vec.getValueCount()); sortedVec.setLastSet(vec.getValueCount() - 1); sortedVec.setValueCount(vec.getValueCount()); @@ -96,14 +93,23 @@ public void testSortString() { assertTrue(sortedVec.isNull(0)); assertTrue(sortedVec.isNull(1)); - assertEquals("12", new String(Objects.requireNonNull(sortedVec.get(2)), StandardCharsets.UTF_8)); - assertEquals("abc", new String(Objects.requireNonNull(sortedVec.get(3)), StandardCharsets.UTF_8)); - assertEquals("dictionary", new String(Objects.requireNonNull(sortedVec.get(4)), StandardCharsets.UTF_8)); - assertEquals("good", new String(Objects.requireNonNull(sortedVec.get(5)), StandardCharsets.UTF_8)); - assertEquals("hello", new String(Objects.requireNonNull(sortedVec.get(6)), StandardCharsets.UTF_8)); - assertEquals("hello", new String(Objects.requireNonNull(sortedVec.get(7)), StandardCharsets.UTF_8)); - assertEquals("world", new String(Objects.requireNonNull(sortedVec.get(8)), StandardCharsets.UTF_8)); - assertEquals("yes", new String(Objects.requireNonNull(sortedVec.get(9)), StandardCharsets.UTF_8)); + assertEquals( + "12", new String(Objects.requireNonNull(sortedVec.get(2)), StandardCharsets.UTF_8)); + assertEquals( + "abc", new String(Objects.requireNonNull(sortedVec.get(3)), StandardCharsets.UTF_8)); + assertEquals( + "dictionary", + new String(Objects.requireNonNull(sortedVec.get(4)), StandardCharsets.UTF_8)); + assertEquals( + "good", new String(Objects.requireNonNull(sortedVec.get(5)), StandardCharsets.UTF_8)); + assertEquals( + "hello", new String(Objects.requireNonNull(sortedVec.get(6)), StandardCharsets.UTF_8)); + assertEquals( + "hello", new String(Objects.requireNonNull(sortedVec.get(7)), StandardCharsets.UTF_8)); + assertEquals( + "world", new String(Objects.requireNonNull(sortedVec.get(8)), StandardCharsets.UTF_8)); + assertEquals( + "yes", new String(Objects.requireNonNull(sortedVec.get(9)), StandardCharsets.UTF_8)); sortedVec.close(); } diff --git a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestVariableWidthSorting.java b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestVariableWidthSorting.java index 7951c39d550d2..5c37ddf9284e4 100644 --- a/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestVariableWidthSorting.java +++ b/java/algorithm/src/test/java/org/apache/arrow/algorithm/sort/TestVariableWidthSorting.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.algorithm.sort; import static org.junit.jupiter.api.Assertions.assertArrayEquals; @@ -28,7 +27,6 @@ import java.util.Comparator; import java.util.List; import java.util.function.Function; - import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.BaseVariableWidthVector; @@ -41,9 +39,7 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -/** - * Test sorting variable width vectors with random data. - */ +/** Test sorting variable width vectors with random data. */ @RunWith(Parameterized.class) public class TestVariableWidthSorting> { @@ -72,8 +68,11 @@ public void shutdown() { } public TestVariableWidthSorting( - int length, double nullFraction, String desc, - Function vectorGenerator, TestSortingUtil.DataGenerator dataGenerator) { + int length, + double nullFraction, + String desc, + Function vectorGenerator, + TestSortingUtil.DataGenerator dataGenerator) { this.length = length; this.nullFraction = nullFraction; this.vectorGenerator = vectorGenerator; @@ -92,9 +91,11 @@ void sortOutOfPlace() { // sort the vector VariableWidthOutOfPlaceVectorSorter sorter = new VariableWidthOutOfPlaceVectorSorter(); - VectorValueComparator comparator = DefaultVectorComparators.createDefaultComparator(vector); + VectorValueComparator comparator = + DefaultVectorComparators.createDefaultComparator(vector); - try (V sortedVec = (V) vector.getField().getFieldType().createNewSingleVector("", allocator, null)) { + try (V sortedVec = + (V) vector.getField().getFieldType().createNewSingleVector("", allocator, null)) { int dataSize = vector.getOffsetBuffer().getInt(vector.getValueCount() * 4L); sortedVec.allocateNew(dataSize, vector.getValueCount()); sortedVec.setValueCount(vector.getValueCount()); @@ -112,33 +113,36 @@ public static Collection getParameters() { List params = new ArrayList<>(); for (int length : VECTOR_LENGTHS) { for (double nullFrac : NULL_FRACTIONS) { - params.add(new Object[]{ - length, nullFrac, "VarCharVector", - (Function) allocator -> new VarCharVector("vector", allocator), - TestSortingUtil.STRING_GENERATOR - }); + params.add( + new Object[] { + length, + nullFrac, + "VarCharVector", + (Function) + allocator -> new VarCharVector("vector", allocator), + TestSortingUtil.STRING_GENERATOR + }); } } return params; } - /** - * Verify results as byte arrays. - */ + /** Verify results as byte arrays. */ public static void verifyResults(V vector, String[] expected) { assertEquals(vector.getValueCount(), expected.length); for (int i = 0; i < expected.length; i++) { if (expected[i] == null) { assertTrue(vector.isNull(i)); } else { - assertArrayEquals(((Text) vector.getObject(i)).getBytes(), expected[i].getBytes(StandardCharsets.UTF_8)); + assertArrayEquals( + ((Text) vector.getObject(i)).getBytes(), expected[i].getBytes(StandardCharsets.UTF_8)); } } } /** - * String comparator with the same behavior as that of - * {@link DefaultVectorComparators.VariableWidthComparator}. + * String comparator with the same behavior as that of {@link + * DefaultVectorComparators.VariableWidthComparator}. */ static class StringComparator implements Comparator { diff --git a/java/dev/checkstyle/checkstyle-spotless.xml b/java/dev/checkstyle/checkstyle-spotless.xml new file mode 100644 index 0000000000000..cbaec1a39bf2c --- /dev/null +++ b/java/dev/checkstyle/checkstyle-spotless.xml @@ -0,0 +1,286 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/java/dev/checkstyle/checkstyle.license b/java/dev/license/asf-java.license similarity index 100% rename from java/dev/checkstyle/checkstyle.license rename to java/dev/license/asf-java.license diff --git a/java/dev/license/asf-xml.license b/java/dev/license/asf-xml.license new file mode 100644 index 0000000000000..a43b97bca8f0f --- /dev/null +++ b/java/dev/license/asf-xml.license @@ -0,0 +1,11 @@ + + \ No newline at end of file diff --git a/java/maven/pom.xml b/java/maven/pom.xml index 72140dd6570d0..8a4043016e770 100644 --- a/java/maven/pom.xml +++ b/java/maven/pom.xml @@ -237,7 +237,7 @@ maven-checkstyle-plugin ../dev/checkstyle/checkstyle.xml - ../dev/checkstyle/checkstyle.license + ../dev/license/asf-java.license ../dev/checkstyle/suppressions.xml true UTF-8 diff --git a/java/pom.xml b/java/pom.xml index 9624444cf422d..bcb8b46843f2d 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -92,12 +92,15 @@ 1.11.3 2 + dev/checkstyle/checkstyle.xml true 9+181-r4173-1 2.28.0 5.11.0 5.2.0 3.43.0 + + **/*.java none -Xdoclint:none @@ -701,8 +704,8 @@ maven-checkstyle-plugin **/module-info.java - dev/checkstyle/checkstyle.xml - dev/checkstyle/checkstyle.license + ${checkstyle.config.location} + dev/license/asf-java.license dev/checkstyle/suppressions.xml true UTF-8 @@ -803,6 +806,19 @@ + + + ${spotless.java.excludes} + + + 1.7 + + + + ${maven.multiModuleProjectDirectory}/dev/license/asf-java.license + package + + @@ -929,7 +945,7 @@ Error Prone 2.10.0 is the latest version to support running on JDK 8. With right flags it could be upgraded, - but we choose to keep this unchanged for now. + but we choose to keep this unchanged for now. --> 2.10.0 From 95db23e9e02c7d724269b0a0f241e53ecfbe76be Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 13:41:04 +0900 Subject: [PATCH 91/93] MINOR: [Java] Bump org.jacoco:jacoco-maven-plugin from 0.8.11 to 0.8.12 in /java (#41516) Bumps [org.jacoco:jacoco-maven-plugin](https://github.com/jacoco/jacoco) from 0.8.11 to 0.8.12.

    Release notes

    Sourced from org.jacoco:jacoco-maven-plugin's releases.

    0.8.12

    New Features

    • JaCoCo now officially supports Java 22 (GitHub #1596).
    • Experimental support for Java 23 class files (GitHub #1553).

    Fixed bugs

    • Branches added by the Kotlin compiler for functions with default arguments and having more than 32 parameters are filtered out during generation of report (GitHub #1556).
    • Branch added by the Kotlin compiler version 1.5.0 and above for reading from lateinit property is filtered out during generation of report (GitHub #1568).

    Non-functional Changes

    • JaCoCo now depends on ASM 9.7 (GitHub #1600).
    Commits

    [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=org.jacoco:jacoco-maven-plugin&package-manager=maven&previous-version=0.8.11&new-version=0.8.12)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) You can trigger a rebase of this PR by commenting `@ dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
    Dependabot commands and options
    You can trigger Dependabot actions by commenting on this PR: - `@ dependabot rebase` will rebase this PR - `@ dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@ dependabot merge` will merge this PR after your CI passes on it - `@ dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@ dependabot cancel merge` will cancel a previously requested merge and block automerging - `@ dependabot reopen` will reopen this PR if it is closed - `@ dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@ dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@ dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@ dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
    > **Note** > Automatic rebases have been disabled on this pull request as it has been open for over 30 days. Authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: David Li --- java/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/pom.xml b/java/pom.xml index bcb8b46843f2d..085546573596a 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -342,7 +342,7 @@ org.jacoco jacoco-maven-plugin - 0.8.11 + 0.8.12 ArrowType.Utf8
    • - *
  • INT --> ArrowType.Int(32, signed)
    • - *
  • LONG --> ArrowType.Int(64, signed)
    • - *
  • FLOAT --> ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)
    • - *
  • DOUBLE --> ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)
    • - *
  • BOOLEAN --> ArrowType.Bool
    • - *
  • BYTES --> ArrowType.Binary
    • - *
  • ARRAY --> ArrowType.List
    • - *
  • MAP --> ArrowType.Map
    • - *
  • FIXED --> ArrowType.FixedSizeBinary
    • - *
  • RECORD --> ArrowType.Struct
    • - *
  • UNION --> ArrowType.Union
    • - *
  • ENUM--> ArrowType.Int
    • - *
  • DECIMAL --> ArrowType.Decimal
    • - *
  • Date --> ArrowType.Date(DateUnit.DAY)
    • - *
  • TimeMillis --> ArrowType.Time(TimeUnit.MILLISECOND, 32)
    • - *
  • TimeMicros --> ArrowType.Time(TimeUnit.MICROSECOND, 64)
    • - *
  • TimestampMillis --> ArrowType.Timestamp(TimeUnit.MILLISECOND, null)
    • - *
  • TimestampMicros --> ArrowType.Timestamp(TimeUnit.MICROSECOND, null)
    • + *
  • STRING --> ArrowType.Utf8 + *
  • INT --> ArrowType.Int(32, signed) + *
  • LONG --> ArrowType.Int(64, signed) + *
  • FLOAT --> ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE) + *
  • DOUBLE --> ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE) + *
  • BOOLEAN --> ArrowType.Bool + *
  • BYTES --> ArrowType.Binary + *
  • ARRAY --> ArrowType.List + *
  • MAP --> ArrowType.Map + *
  • FIXED --> ArrowType.FixedSizeBinary + *
  • RECORD --> ArrowType.Struct + *
  • UNION --> ArrowType.Union + *
  • ENUM--> ArrowType.Int + *
  • DECIMAL --> ArrowType.Decimal + *
  • Date --> ArrowType.Date(DateUnit.DAY) + *
  • TimeMillis --> ArrowType.Time(TimeUnit.MILLISECOND, 32) + *
  • TimeMicros --> ArrowType.Time(TimeUnit.MICROSECOND, 64) + *
  • TimestampMillis --> ArrowType.Timestamp(TimeUnit.MILLISECOND, null) + *
  • TimestampMicros --> ArrowType.Timestamp(TimeUnit.MICROSECOND, null) * */ - private static Consumer createConsumer(Schema schema, String name, AvroToArrowConfig config) { return createConsumer(schema, name, false, config, null); } - private static Consumer createConsumer(Schema schema, String name, AvroToArrowConfig config, FieldVector vector) { + private static Consumer createConsumer( + Schema schema, String name, AvroToArrowConfig config, FieldVector vector) { return createConsumer(schema, name, false, config, vector); } @@ -144,7 +143,8 @@ private static Consumer createConsumer(Schema schema, String name, AvroToArrowCo * * @param schema avro schema * @param name arrow field name - * @param consumerVector vector to keep in consumer, if v == null, will create a new vector via field. + * @param consumerVector vector to keep in consumer, if v == null, will create a new vector via + * field. * @return consumer */ private static Consumer createConsumer( @@ -185,7 +185,7 @@ private static Consumer createConsumer( break; case STRING: arrowType = new ArrowType.Utf8(); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/null, getMetaData(schema)); + fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroStringConsumer((VarCharVector) vector); break; @@ -193,12 +193,18 @@ private static Consumer createConsumer( Map extProps = createExternalProps(schema); if (logicalType instanceof LogicalTypes.Decimal) { arrowType = createDecimalArrowType((LogicalTypes.Decimal) logicalType); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/null, getMetaData(schema, extProps)); + fieldType = + new FieldType( + nullable, arrowType, /*dictionary=*/ null, getMetaData(schema, extProps)); vector = createVector(consumerVector, fieldType, name, allocator); - consumer = new AvroDecimalConsumer.FixedDecimalConsumer((DecimalVector) vector, schema.getFixedSize()); + consumer = + new AvroDecimalConsumer.FixedDecimalConsumer( + (DecimalVector) vector, schema.getFixedSize()); } else { arrowType = new ArrowType.FixedSizeBinary(schema.getFixedSize()); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/null, getMetaData(schema, extProps)); + fieldType = + new FieldType( + nullable, arrowType, /*dictionary=*/ null, getMetaData(schema, extProps)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroFixedConsumer((FixedSizeBinaryVector) vector, schema.getFixedSize()); } @@ -206,84 +212,85 @@ private static Consumer createConsumer( case INT: if (logicalType instanceof LogicalTypes.Date) { arrowType = new ArrowType.Date(DateUnit.DAY); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/null, getMetaData(schema)); + fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroDateConsumer((DateDayVector) vector); } else if (logicalType instanceof LogicalTypes.TimeMillis) { arrowType = new ArrowType.Time(TimeUnit.MILLISECOND, 32); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/null, getMetaData(schema)); + fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroTimeMillisConsumer((TimeMilliVector) vector); } else { - arrowType = new ArrowType.Int(32, /*isSigned=*/true); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/null, getMetaData(schema)); + arrowType = new ArrowType.Int(32, /*isSigned=*/ true); + fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroIntConsumer((IntVector) vector); } break; case BOOLEAN: arrowType = new ArrowType.Bool(); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/null, getMetaData(schema)); + fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroBooleanConsumer((BitVector) vector); break; case LONG: if (logicalType instanceof LogicalTypes.TimeMicros) { arrowType = new ArrowType.Time(TimeUnit.MICROSECOND, 64); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/null, getMetaData(schema)); + fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroTimeMicroConsumer((TimeMicroVector) vector); } else if (logicalType instanceof LogicalTypes.TimestampMillis) { arrowType = new ArrowType.Timestamp(TimeUnit.MILLISECOND, null); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/null, getMetaData(schema)); + fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroTimestampMillisConsumer((TimeStampMilliVector) vector); } else if (logicalType instanceof LogicalTypes.TimestampMicros) { arrowType = new ArrowType.Timestamp(TimeUnit.MICROSECOND, null); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/null, getMetaData(schema)); + fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroTimestampMicrosConsumer((TimeStampMicroVector) vector); } else { - arrowType = new ArrowType.Int(64, /*isSigned=*/true); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/null, getMetaData(schema)); + arrowType = new ArrowType.Int(64, /*isSigned=*/ true); + fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroLongConsumer((BigIntVector) vector); } break; case FLOAT: arrowType = new ArrowType.FloatingPoint(SINGLE); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/null, getMetaData(schema)); + fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroFloatConsumer((Float4Vector) vector); break; case DOUBLE: arrowType = new ArrowType.FloatingPoint(DOUBLE); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/null, getMetaData(schema)); + fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroDoubleConsumer((Float8Vector) vector); break; case BYTES: if (logicalType instanceof LogicalTypes.Decimal) { arrowType = createDecimalArrowType((LogicalTypes.Decimal) logicalType); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/null, getMetaData(schema)); + fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroDecimalConsumer.BytesDecimalConsumer((DecimalVector) vector); } else { arrowType = new ArrowType.Binary(); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/null, getMetaData(schema)); + fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); vector = createVector(consumerVector, fieldType, name, allocator); consumer = new AvroBytesConsumer((VarBinaryVector) vector); } break; case NULL: arrowType = new ArrowType.Null(); - fieldType = new FieldType(nullable, arrowType, /*dictionary=*/null, getMetaData(schema)); - vector = fieldType.createNewSingleVector(name, allocator, /*schemaCallBack=*/null); + fieldType = new FieldType(nullable, arrowType, /*dictionary=*/ null, getMetaData(schema)); + vector = fieldType.createNewSingleVector(name, allocator, /*schemaCallBack=*/ null); consumer = new AvroNullConsumer((NullVector) vector); break; default: // no-op, shouldn't get here - throw new UnsupportedOperationException("Can't convert avro type %s to arrow type." + type.getName()); + throw new UnsupportedOperationException( + "Can't convert avro type %s to arrow type." + type.getName()); } return consumer; } @@ -291,15 +298,16 @@ private static Consumer createConsumer( private static ArrowType createDecimalArrowType(LogicalTypes.Decimal logicalType) { final int scale = logicalType.getScale(); final int precision = logicalType.getPrecision(); - Preconditions.checkArgument(precision > 0 && precision <= 38, - "Precision must be in range of 1 to 38"); - Preconditions.checkArgument(scale >= 0 && scale <= 38, - "Scale must be in range of 0 to 38."); - Preconditions.checkArgument(scale <= precision, - "Invalid decimal scale: %s (greater than precision: %s)", scale, precision); + Preconditions.checkArgument( + precision > 0 && precision <= 38, "Precision must be in range of 1 to 38"); + Preconditions.checkArgument(scale >= 0 && scale <= 38, "Scale must be in range of 0 to 38."); + Preconditions.checkArgument( + scale <= precision, + "Invalid decimal scale: %s (greater than precision: %s)", + scale, + precision); return new ArrowType.Decimal(precision, scale, 128); - } private static Consumer createSkipConsumer(Schema schema) { @@ -309,41 +317,46 @@ private static Consumer createSkipConsumer(Schema schema) { switch (type) { case UNION: - List unionDelegates = schema.getTypes().stream().map(s -> - createSkipConsumer(s)).collect(Collectors.toList()); + List unionDelegates = + schema.getTypes().stream().map(s -> createSkipConsumer(s)).collect(Collectors.toList()); skipFunction = decoder -> unionDelegates.get(decoder.readInt()).consume(decoder); break; case ARRAY: Consumer elementDelegate = createSkipConsumer(schema.getElementType()); - skipFunction = decoder -> { - for (long i = decoder.skipArray(); i != 0; i = decoder.skipArray()) { - for (long j = 0; j < i; j++) { - elementDelegate.consume(decoder); - } - } - }; + skipFunction = + decoder -> { + for (long i = decoder.skipArray(); i != 0; i = decoder.skipArray()) { + for (long j = 0; j < i; j++) { + elementDelegate.consume(decoder); + } + } + }; break; case MAP: Consumer valueDelegate = createSkipConsumer(schema.getValueType()); - skipFunction = decoder -> { - for (long i = decoder.skipMap(); i != 0; i = decoder.skipMap()) { - for (long j = 0; j < i; j++) { - decoder.skipString(); // Discard key - valueDelegate.consume(decoder); - } - } - }; + skipFunction = + decoder -> { + for (long i = decoder.skipMap(); i != 0; i = decoder.skipMap()) { + for (long j = 0; j < i; j++) { + decoder.skipString(); // Discard key + valueDelegate.consume(decoder); + } + } + }; break; case RECORD: - List delegates = schema.getFields().stream().map(field -> - createSkipConsumer(field.schema())).collect(Collectors.toList()); + List delegates = + schema.getFields().stream() + .map(field -> createSkipConsumer(field.schema())) + .collect(Collectors.toList()); - skipFunction = decoder -> { - for (Consumer consumer : delegates) { - consumer.consume(decoder); - } - }; + skipFunction = + decoder -> { + for (Consumer consumer : delegates) { + consumer.consume(decoder); + } + }; break; case ENUM: @@ -374,7 +387,7 @@ private static Consumer createSkipConsumer(Schema schema) { skipFunction = decoder -> decoder.skipBytes(); break; case NULL: - skipFunction = decoder -> { }; + skipFunction = decoder -> {}; break; default: // no-op, shouldn't get here @@ -384,8 +397,7 @@ private static Consumer createSkipConsumer(Schema schema) { return new SkipConsumer(skipFunction); } - static CompositeAvroConsumer createCompositeConsumer( - Schema schema, AvroToArrowConfig config) { + static CompositeAvroConsumer createCompositeConsumer(Schema schema, AvroToArrowConfig config) { List consumers = new ArrayList<>(); final Set skipFieldNames = config.getSkipFieldNames(); @@ -399,7 +411,6 @@ static CompositeAvroConsumer createCompositeConsumer( Consumer consumer = createConsumer(field.schema(), field.name(), config); consumers.add(consumer); } - } } else { Consumer consumer = createConsumer(schema, "", config); @@ -409,9 +420,11 @@ static CompositeAvroConsumer createCompositeConsumer( return new CompositeAvroConsumer(consumers); } - private static FieldVector createVector(FieldVector consumerVector, FieldType fieldType, - String name, BufferAllocator allocator) { - return consumerVector != null ? consumerVector : fieldType.createNewSingleVector(name, allocator, null); + private static FieldVector createVector( + FieldVector consumerVector, FieldType fieldType, String name, BufferAllocator allocator) { + return consumerVector != null + ? consumerVector + : fieldType.createNewSingleVector(name, allocator, null); } private static String getDefaultFieldName(ArrowType type) { @@ -424,10 +437,7 @@ private static Field avroSchemaToField(Schema schema, String name, AvroToArrowCo } private static Field avroSchemaToField( - Schema schema, - String name, - AvroToArrowConfig config, - Map externalProps) { + Schema schema, String name, AvroToArrowConfig config, Map externalProps) { final Schema.Type type = schema.getType(); final LogicalType logicalType = schema.getLogicalType(); @@ -441,7 +451,8 @@ private static Field avroSchemaToField( // Union child vector should use default name children.add(avroSchemaToField(childSchema, null, config)); } - fieldType = createFieldType(new ArrowType.Union(UnionMode.Sparse, null), schema, externalProps); + fieldType = + createFieldType(new ArrowType.Union(UnionMode.Sparse, null), schema, externalProps); break; case ARRAY: Schema elementSchema = schema.getElementType(); @@ -450,14 +461,18 @@ private static Field avroSchemaToField( break; case MAP: // MapVector internal struct field and key field should be non-nullable - FieldType keyFieldType = new FieldType(/*nullable=*/false, new ArrowType.Utf8(), /*dictionary=*/null); - Field keyField = new Field("key", keyFieldType, /*children=*/null); + FieldType keyFieldType = + new FieldType(/*nullable=*/ false, new ArrowType.Utf8(), /*dictionary=*/ null); + Field keyField = new Field("key", keyFieldType, /*children=*/ null); Field valueField = avroSchemaToField(schema.getValueType(), "value", config); - FieldType structFieldType = new FieldType(false, new ArrowType.Struct(), /*dictionary=*/null); - Field structField = new Field("internal", structFieldType, Arrays.asList(keyField, valueField)); + FieldType structFieldType = + new FieldType(false, new ArrowType.Struct(), /*dictionary=*/ null); + Field structField = + new Field("internal", structFieldType, Arrays.asList(keyField, valueField)); children.add(structField); - fieldType = createFieldType(new ArrowType.Map(/*keysSorted=*/false), schema, externalProps); + fieldType = + createFieldType(new ArrowType.Map(/*keysSorted=*/ false), schema, externalProps); break; case RECORD: final Set skipFieldNames = config.getSkipFieldNames(); @@ -486,8 +501,12 @@ private static Field avroSchemaToField( int enumCount = schema.getEnumSymbols().size(); ArrowType.Int indexType = DictionaryEncoder.getIndexType(enumCount); - fieldType = createFieldType(indexType, schema, externalProps, - new DictionaryEncoding(current, /*ordered=*/false, /*indexType=*/indexType)); + fieldType = + createFieldType( + indexType, + schema, + externalProps, + new DictionaryEncoding(current, /*ordered=*/ false, /*indexType=*/ indexType)); break; case STRING: @@ -509,7 +528,7 @@ private static Field avroSchemaToField( } else if (logicalType instanceof LogicalTypes.TimeMillis) { intArrowType = new ArrowType.Time(TimeUnit.MILLISECOND, 32); } else { - intArrowType = new ArrowType.Int(32, /*isSigned=*/true); + intArrowType = new ArrowType.Int(32, /*isSigned=*/ true); } fieldType = createFieldType(intArrowType, schema, externalProps); break; @@ -525,7 +544,7 @@ private static Field avroSchemaToField( } else if (logicalType instanceof LogicalTypes.TimestampMicros) { longArrowType = new ArrowType.Timestamp(TimeUnit.MICROSECOND, null); } else { - longArrowType = new ArrowType.Int(64, /*isSigned=*/true); + longArrowType = new ArrowType.Int(64, /*isSigned=*/ true); } fieldType = createFieldType(longArrowType, schema, externalProps); break; @@ -558,8 +577,8 @@ private static Field avroSchemaToField( return new Field(name, fieldType, children.size() == 0 ? null : children); } - private static Consumer createArrayConsumer(Schema schema, String name, AvroToArrowConfig config, - FieldVector consumerVector) { + private static Consumer createArrayConsumer( + Schema schema, String name, AvroToArrowConfig config, FieldVector consumerVector) { ListVector listVector; if (consumerVector == null) { @@ -578,8 +597,8 @@ private static Consumer createArrayConsumer(Schema schema, String name, AvroToAr return new AvroArraysConsumer(listVector, delegate); } - private static Consumer createStructConsumer(Schema schema, String name, AvroToArrowConfig config, - FieldVector consumerVector) { + private static Consumer createStructConsumer( + Schema schema, String name, AvroToArrowConfig config, FieldVector consumerVector) { final Set skipFieldNames = config.getSkipFieldNames(); @@ -601,19 +620,22 @@ private static Consumer createStructConsumer(Schema schema, String name, AvroToA if (skipFieldNames.contains(fullChildName)) { delegate = createSkipConsumer(childField.schema()); } else { - delegate = createConsumer(childField.schema(), fullChildName, config, - structVector.getChildrenFromFields().get(vectorIndex++)); + delegate = + createConsumer( + childField.schema(), + fullChildName, + config, + structVector.getChildrenFromFields().get(vectorIndex++)); } delegates[i] = delegate; } return new AvroStructConsumer(structVector, delegates); - } - private static Consumer createEnumConsumer(Schema schema, String name, AvroToArrowConfig config, - FieldVector consumerVector) { + private static Consumer createEnumConsumer( + Schema schema, String name, AvroToArrowConfig config, FieldVector consumerVector) { BaseIntVector indexVector; if (consumerVector == null) { @@ -630,16 +652,14 @@ private static Consumer createEnumConsumer(Schema schema, String name, AvroToArr for (int i = 0; i < valueCount; i++) { dictVector.set(i, schema.getEnumSymbols().get(i).getBytes(StandardCharsets.UTF_8)); } - Dictionary dictionary = - new Dictionary(dictVector, indexVector.getField().getDictionary()); + Dictionary dictionary = new Dictionary(dictVector, indexVector.getField().getDictionary()); config.getProvider().put(dictionary); return new AvroEnumConsumer(indexVector); - } - private static Consumer createMapConsumer(Schema schema, String name, AvroToArrowConfig config, - FieldVector consumerVector) { + private static Consumer createMapConsumer( + Schema schema, String name, AvroToArrowConfig config, FieldVector consumerVector) { MapVector mapVector; if (consumerVector == null) { @@ -653,10 +673,14 @@ private static Consumer createMapConsumer(Schema schema, String name, AvroToArro StructVector structVector = (StructVector) mapVector.getDataVector(); // keys in avro map are always assumed to be strings. - Consumer keyConsumer = new AvroStringConsumer( - (VarCharVector) structVector.getChildrenFromFields().get(0)); - Consumer valueConsumer = createConsumer(schema.getValueType(), schema.getValueType().getName(), - config, structVector.getChildrenFromFields().get(1)); + Consumer keyConsumer = + new AvroStringConsumer((VarCharVector) structVector.getChildrenFromFields().get(0)); + Consumer valueConsumer = + createConsumer( + schema.getValueType(), + schema.getValueType().getName(), + config, + structVector.getChildrenFromFields().get(1)); AvroStructConsumer internalConsumer = new AvroStructConsumer(structVector, new Consumer[] {keyConsumer, valueConsumer}); @@ -664,11 +688,12 @@ private static Consumer createMapConsumer(Schema schema, String name, AvroToArro return new AvroMapConsumer(mapVector, internalConsumer); } - private static Consumer createUnionConsumer(Schema schema, String name, AvroToArrowConfig config, - FieldVector consumerVector) { + private static Consumer createUnionConsumer( + Schema schema, String name, AvroToArrowConfig config, FieldVector consumerVector) { final int size = schema.getTypes().size(); - final boolean nullable = schema.getTypes().stream().anyMatch(t -> t.getType() == Schema.Type.NULL); + final boolean nullable = + schema.getTypes().stream().anyMatch(t -> t.getType() == Schema.Type.NULL); UnionVector unionVector; if (consumerVector == null) { @@ -695,14 +720,12 @@ private static Consumer createUnionConsumer(Schema schema, String name, AvroToAr /** * Read data from {@link Decoder} and generate a {@link VectorSchemaRoot}. + * * @param schema avro schema * @param decoder avro decoder to read data from */ static VectorSchemaRoot avroToArrowVectors( - Schema schema, - Decoder decoder, - AvroToArrowConfig config) - throws IOException { + Schema schema, Decoder decoder, AvroToArrowConfig config) throws IOException { List vectors = new ArrayList<>(); List consumers = new ArrayList<>(); @@ -726,8 +749,8 @@ static VectorSchemaRoot avroToArrowVectors( } long validConsumerCount = consumers.stream().filter(c -> !c.skippable()).count(); - Preconditions.checkArgument(vectors.size() == validConsumerCount, - "vectors size not equals consumers size."); + Preconditions.checkArgument( + vectors.size() == validConsumerCount, "vectors size not equals consumers size."); List fields = vectors.stream().map(t -> t.getField()).collect(Collectors.toList()); @@ -767,9 +790,7 @@ private static Map getMetaData(Schema schema, Map createExternalProps(Schema schema) { final Map extProps = new HashMap<>(); String doc = schema.getDoc(); @@ -783,8 +804,9 @@ private static Map createExternalProps(Schema schema) { return extProps; } - private static FieldType createFieldType(ArrowType arrowType, Schema schema, Map externalProps) { - return createFieldType(arrowType, schema, externalProps, /*dictionary=*/null); + private static FieldType createFieldType( + ArrowType arrowType, Schema schema, Map externalProps) { + return createFieldType(arrowType, schema, externalProps, /*dictionary=*/ null); } private static FieldType createFieldType( @@ -793,8 +815,8 @@ private static FieldType createFieldType( Map externalProps, DictionaryEncoding dictionary) { - return new FieldType(/*nullable=*/false, arrowType, dictionary, - getMetaData(schema, externalProps)); + return new FieldType( + /*nullable=*/ false, arrowType, dictionary, getMetaData(schema, externalProps)); } private static String convertAliases(Set aliases) { diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowVectorIterator.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowVectorIterator.java index 9a0cfd97a49a1..4123370061794 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowVectorIterator.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/AvroToArrowVectorIterator.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro; import java.io.EOFException; @@ -22,7 +21,6 @@ import java.util.Iterator; import java.util.List; import java.util.stream.Collectors; - import org.apache.arrow.adapter.avro.consumers.CompositeAvroConsumer; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.FieldVector; @@ -32,9 +30,7 @@ import org.apache.avro.Schema; import org.apache.avro.io.Decoder; -/** - * VectorSchemaRoot iterator for partially converting avro data. - */ +/** VectorSchemaRoot iterator for partially converting avro data. */ public class AvroToArrowVectorIterator implements Iterator, AutoCloseable { public static final int NO_LIMIT_BATCH_SIZE = -1; @@ -53,28 +49,18 @@ public class AvroToArrowVectorIterator implements Iterator, Au private final int targetBatchSize; - /** - * Construct an instance. - */ - private AvroToArrowVectorIterator( - Decoder decoder, - Schema schema, - AvroToArrowConfig config) { + /** Construct an instance. */ + private AvroToArrowVectorIterator(Decoder decoder, Schema schema, AvroToArrowConfig config) { this.decoder = decoder; this.schema = schema; this.config = config; this.targetBatchSize = config.getTargetBatchSize(); - } - /** - * Create a ArrowVectorIterator to partially convert data. - */ + /** Create a ArrowVectorIterator to partially convert data. */ public static AvroToArrowVectorIterator create( - Decoder decoder, - Schema schema, - AvroToArrowConfig config) { + Decoder decoder, Schema schema, AvroToArrowConfig config) { AvroToArrowVectorIterator iterator = new AvroToArrowVectorIterator(decoder, schema, config); try { @@ -136,9 +122,10 @@ private void load(VectorSchemaRoot root) { ValueVectorUtility.preAllocate(root, targetBatchSize); } - long validConsumerCount = compositeConsumer.getConsumers().stream().filter(c -> - !c.skippable()).count(); - Preconditions.checkArgument(root.getFieldVectors().size() == validConsumerCount, + long validConsumerCount = + compositeConsumer.getConsumers().stream().filter(c -> !c.skippable()).count(); + Preconditions.checkArgument( + root.getFieldVectors().size() == validConsumerCount, "Schema root vectors size not equals to consumers size."); compositeConsumer.resetConsumerVectors(root); @@ -159,9 +146,7 @@ public boolean hasNext() { return nextBatch != null; } - /** - * Gets the next vector. The user is responsible for freeing its resources. - */ + /** Gets the next vector. The user is responsible for freeing its resources. */ @Override public VectorSchemaRoot next() { Preconditions.checkArgument(hasNext()); @@ -175,9 +160,7 @@ public VectorSchemaRoot next() { return returned; } - /** - * Clean up resources. - */ + /** Clean up resources. */ @Override public void close() { if (nextBatch != null) { diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroArraysConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroArraysConsumer.java index fd25986c32b95..4555ce7a295f7 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroArraysConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroArraysConsumer.java @@ -14,25 +14,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; - import org.apache.arrow.vector.complex.ListVector; import org.apache.avro.io.Decoder; /** - * Consumer which consume array type values from avro decoder. - * Write the data to {@link ListVector}. + * Consumer which consume array type values from avro decoder. Write the data to {@link ListVector}. */ public class AvroArraysConsumer extends BaseAvroConsumer { private final Consumer delegate; - /** - * Instantiate a ArrayConsumer. - */ + /** Instantiate a ArrayConsumer. */ public AvroArraysConsumer(ListVector vector, Consumer delegate) { super(vector); this.delegate = delegate; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroBooleanConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroBooleanConsumer.java index bf41828d19f7a..09eb5f3b255d5 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroBooleanConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroBooleanConsumer.java @@ -14,23 +14,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; - import org.apache.arrow.vector.BitVector; import org.apache.avro.io.Decoder; /** - * Consumer which consume boolean type values from avro decoder. - * Write the data to {@link BitVector}. + * Consumer which consume boolean type values from avro decoder. Write the data to {@link + * BitVector}. */ public class AvroBooleanConsumer extends BaseAvroConsumer { - /** - * Instantiate a AvroBooleanConsumer. - */ + /** Instantiate a AvroBooleanConsumer. */ public AvroBooleanConsumer(BitVector vector) { super(vector); } diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroBytesConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroBytesConsumer.java index c8370e480608d..86b6cbb13d881 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroBytesConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroBytesConsumer.java @@ -14,26 +14,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; import java.nio.ByteBuffer; - import org.apache.arrow.vector.VarBinaryVector; import org.apache.avro.io.Decoder; /** - * Consumer which consume bytes type values from avro decoder. - * Write the data to {@link VarBinaryVector}. + * Consumer which consume bytes type values from avro decoder. Write the data to {@link + * VarBinaryVector}. */ public class AvroBytesConsumer extends BaseAvroConsumer { private ByteBuffer cacheBuffer; - /** - * Instantiate a AvroBytesConsumer. - */ + /** Instantiate a AvroBytesConsumer. */ public AvroBytesConsumer(VarBinaryVector vector) { super(vector); } diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroDoubleConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroDoubleConsumer.java index 7cc7dd33b15a9..011cbccc09c5b 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroDoubleConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroDoubleConsumer.java @@ -14,23 +14,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; - import org.apache.arrow.vector.Float8Vector; import org.apache.avro.io.Decoder; /** - * Consumer which consume double type values from avro decoder. - * Write the data to {@link Float8Vector}. + * Consumer which consume double type values from avro decoder. Write the data to {@link + * Float8Vector}. */ public class AvroDoubleConsumer extends BaseAvroConsumer { - /** - * Instantiate a AvroDoubleConsumer. - */ + /** Instantiate a AvroDoubleConsumer. */ public AvroDoubleConsumer(Float8Vector vector) { super(vector); } diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroEnumConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroEnumConsumer.java index 32a2c85f6fc50..f47988fb962a1 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroEnumConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroEnumConsumer.java @@ -14,24 +14,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; - import org.apache.arrow.vector.BaseIntVector; import org.apache.arrow.vector.IntVector; import org.apache.avro.io.Decoder; /** - * Consumer which consume enum type values from avro decoder. - * Write the data to {@link IntVector}. + * Consumer which consume enum type values from avro decoder. Write the data to {@link IntVector}. */ public class AvroEnumConsumer extends BaseAvroConsumer { - /** - * Instantiate a AvroEnumConsumer. - */ + /** Instantiate a AvroEnumConsumer. */ public AvroEnumConsumer(BaseIntVector vector) { super(vector); } diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroFixedConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroFixedConsumer.java index 16b70898fd36a..6b78afd3c95d4 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroFixedConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroFixedConsumer.java @@ -14,25 +14,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; - import org.apache.arrow.vector.FixedSizeBinaryVector; import org.apache.avro.io.Decoder; /** - * Consumer which consume fixed type values from avro decoder. - * Write the data to {@link org.apache.arrow.vector.FixedSizeBinaryVector}. + * Consumer which consume fixed type values from avro decoder. Write the data to {@link + * org.apache.arrow.vector.FixedSizeBinaryVector}. */ public class AvroFixedConsumer extends BaseAvroConsumer { private final byte[] reuseBytes; - /** - * Instantiate a AvroFixedConsumer. - */ + /** Instantiate a AvroFixedConsumer. */ public AvroFixedConsumer(FixedSizeBinaryVector vector, int size) { super(vector); reuseBytes = new byte[size]; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroFloatConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroFloatConsumer.java index b09d2881875b6..2c6d4aa5a05f6 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroFloatConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroFloatConsumer.java @@ -14,23 +14,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; - import org.apache.arrow.vector.Float4Vector; import org.apache.avro.io.Decoder; /** - * Consumer which consume float type values from avro decoder. - * Write the data to {@link Float4Vector}. + * Consumer which consume float type values from avro decoder. Write the data to {@link + * Float4Vector}. */ public class AvroFloatConsumer extends BaseAvroConsumer { - /** - * Instantiate a AvroFloatConsumer. - */ + /** Instantiate a AvroFloatConsumer. */ public AvroFloatConsumer(Float4Vector vector) { super(vector); } diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroIntConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroIntConsumer.java index ae5a2719c5642..22c7b10aa65f7 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroIntConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroIntConsumer.java @@ -14,23 +14,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; - import org.apache.arrow.vector.IntVector; import org.apache.avro.io.Decoder; /** - * Consumer which consume int type values from avro decoder. - * Write the data to {@link IntVector}. + * Consumer which consume int type values from avro decoder. Write the data to {@link IntVector}. */ public class AvroIntConsumer extends BaseAvroConsumer { - /** - * Instantiate a AvroIntConsumer. - */ + /** Instantiate a AvroIntConsumer. */ public AvroIntConsumer(IntVector vector) { super(vector); } diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroLongConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroLongConsumer.java index 4db836acc4586..90c5313417d7c 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroLongConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroLongConsumer.java @@ -14,23 +14,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; - import org.apache.arrow.vector.BigIntVector; import org.apache.avro.io.Decoder; /** - * Consumer which consume long type values from avro decoder. - * Write the data to {@link BigIntVector}. + * Consumer which consume long type values from avro decoder. Write the data to {@link + * BigIntVector}. */ public class AvroLongConsumer extends BaseAvroConsumer { - /** - * Instantiate a AvroLongConsumer. - */ + /** Instantiate a AvroLongConsumer. */ public AvroLongConsumer(BigIntVector vector) { super(vector); } diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroMapConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroMapConsumer.java index 1ea97e63b61e5..543471533ec01 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroMapConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroMapConsumer.java @@ -14,27 +14,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; - import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.StructVector; import org.apache.avro.io.Decoder; /** - * Consumer which consume map type values from avro decoder. - * Write the data to {@link MapVector}. + * Consumer which consume map type values from avro decoder. Write the data to {@link MapVector}. */ public class AvroMapConsumer extends BaseAvroConsumer { private final Consumer delegate; - /** - * Instantiate a AvroMapConsumer. - */ + /** Instantiate a AvroMapConsumer. */ public AvroMapConsumer(MapVector vector, Consumer delegate) { super(vector); this.delegate = delegate; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroNullConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroNullConsumer.java index 4c7bb8c03bad3..0f80c2b7b2db3 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroNullConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroNullConsumer.java @@ -14,17 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; - import org.apache.arrow.vector.NullVector; import org.apache.avro.io.Decoder; /** - * Consumer which consume null type values from avro decoder. - * Corresponding to {@link org.apache.arrow.vector.NullVector}. + * Consumer which consume null type values from avro decoder. Corresponding to {@link + * org.apache.arrow.vector.NullVector}. */ public class AvroNullConsumer extends BaseAvroConsumer { diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroStringConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroStringConsumer.java index 072270aa6c081..164d595e9c6ac 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroStringConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroStringConsumer.java @@ -14,26 +14,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; import java.nio.ByteBuffer; - import org.apache.arrow.vector.VarCharVector; import org.apache.avro.io.Decoder; /** - * Consumer which consume string type values from avro decoder. - * Write the data to {@link VarCharVector}. + * Consumer which consume string type values from avro decoder. Write the data to {@link + * VarCharVector}. */ public class AvroStringConsumer extends BaseAvroConsumer { private ByteBuffer cacheBuffer; - /** - * Instantiate a AvroStringConsumer. - */ + /** Instantiate a AvroStringConsumer. */ public AvroStringConsumer(VarCharVector vector) { super(vector); } diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroStructConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroStructConsumer.java index a02b1577f9fa8..94c2f611e84b7 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroStructConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroStructConsumer.java @@ -14,27 +14,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; - import org.apache.arrow.util.AutoCloseables; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.complex.StructVector; import org.apache.avro.io.Decoder; /** - * Consumer which consume nested record type values from avro decoder. - * Write the data to {@link org.apache.arrow.vector.complex.StructVector}. + * Consumer which consume nested record type values from avro decoder. Write the data to {@link + * org.apache.arrow.vector.complex.StructVector}. */ public class AvroStructConsumer extends BaseAvroConsumer { private final Consumer[] delegates; - /** - * Instantiate a AvroStructConsumer. - */ + /** Instantiate a AvroStructConsumer. */ public AvroStructConsumer(StructVector vector, Consumer[] delegates) { super(vector); this.delegates = delegates; @@ -49,7 +45,6 @@ public void consume(Decoder decoder) throws IOException { } vector.setIndexDefined(currentIndex); currentIndex++; - } @Override diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroUnionsConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroUnionsConsumer.java index 76287543b0646..5a8e23e62892c 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroUnionsConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/AvroUnionsConsumer.java @@ -14,11 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; - import org.apache.arrow.util.AutoCloseables; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.complex.UnionVector; @@ -26,17 +24,15 @@ import org.apache.avro.io.Decoder; /** - * Consumer which consume unions type values from avro decoder. - * Write the data to {@link org.apache.arrow.vector.complex.UnionVector}. + * Consumer which consume unions type values from avro decoder. Write the data to {@link + * org.apache.arrow.vector.complex.UnionVector}. */ public class AvroUnionsConsumer extends BaseAvroConsumer { private Consumer[] delegates; private Types.MinorType[] types; - /** - * Instantiate an AvroUnionConsumer. - */ + /** Instantiate an AvroUnionConsumer. */ public AvroUnionsConsumer(UnionVector vector, Consumer[] delegates, Types.MinorType[] types) { super(vector); @@ -53,7 +49,8 @@ public void consume(Decoder decoder) throws IOException { vector.setType(currentIndex, types[fieldIndex]); // In UnionVector we need to set sub vector writer position before consume a value - // because in the previous iterations we might not have written to the specific union sub vector. + // because in the previous iterations we might not have written to the specific union sub + // vector. delegate.setPosition(currentIndex); delegate.consume(decoder); diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/BaseAvroConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/BaseAvroConsumer.java index 66a6cda68401e..9430d83cb4372 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/BaseAvroConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/BaseAvroConsumer.java @@ -14,13 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro.consumers; import org.apache.arrow.vector.FieldVector; /** * Base class for non-skippable avro consumers. + * * @param vector type. */ public abstract class BaseAvroConsumer implements Consumer { @@ -30,6 +30,7 @@ public abstract class BaseAvroConsumer implements Consume /** * Constructs a base avro consumer. + * * @param vector the vector to consume. */ public BaseAvroConsumer(T vector) { diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/CompositeAvroConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/CompositeAvroConsumer.java index 97812226180ac..11c1f7712ef19 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/CompositeAvroConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/CompositeAvroConsumer.java @@ -14,20 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; import java.util.List; - import org.apache.arrow.util.AutoCloseables; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.avro.io.Decoder; -/** - * Composite consumer which hold all consumers. - * It manages the consume and cleanup process. - */ +/** Composite consumer which hold all consumers. It manages the consume and cleanup process. */ public class CompositeAvroConsumer implements AutoCloseable { private final List consumers; @@ -40,18 +35,14 @@ public CompositeAvroConsumer(List consumers) { this.consumers = consumers; } - /** - * Consume decoder data. - */ + /** Consume decoder data. */ public void consume(Decoder decoder) throws IOException { for (Consumer consumer : consumers) { consumer.consume(decoder); } } - /** - * Reset vector of consumers with the given {@link VectorSchemaRoot}. - */ + /** Reset vector of consumers with the given {@link VectorSchemaRoot}. */ public void resetConsumerVectors(VectorSchemaRoot root) { int index = 0; for (Consumer consumer : consumers) { diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/Consumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/Consumer.java index 8eaaf74cff68a..0c07f90bf5f39 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/Consumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/Consumer.java @@ -14,59 +14,49 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; - import org.apache.arrow.vector.FieldVector; import org.apache.avro.io.Decoder; /** * Interface that is used to consume values from avro decoder. + * * @param The vector within consumer or its delegate, used for partially consume purpose. */ public interface Consumer extends AutoCloseable { /** * Consume a specific type value from avro decoder and write it to vector. + * * @param decoder avro decoder to read data * @throws IOException on error */ void consume(Decoder decoder) throws IOException; - /** - * Add null value to vector by making writer position + 1. - */ + /** Add null value to vector by making writer position + 1. */ void addNull(); - /** - * Set the position to write value into vector. - */ + /** Set the position to write value into vector. */ void setPosition(int index); - /** - * Get the vector within the consumer. - */ + /** Get the vector within the consumer. */ FieldVector getVector(); - /** - * Close this consumer when occurs exception to avoid potential leak. - */ + /** Close this consumer when occurs exception to avoid potential leak. */ @Override void close() throws Exception; /** * Reset the vector within consumer for partial read purpose. + * * @return true if reset is successful, false if reset is not needed. */ boolean resetValueVector(T vector); - /** - * Indicates whether the consumer is type of {@link SkipConsumer}. - */ + /** Indicates whether the consumer is type of {@link SkipConsumer}. */ default boolean skippable() { return false; } - } diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/SkipConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/SkipConsumer.java index 1ac0a6d71557b..2c104728ce620 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/SkipConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/SkipConsumer.java @@ -14,17 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; - import org.apache.arrow.vector.FieldVector; import org.apache.avro.io.Decoder; -/** - * Consumer which skip (throw away) data from the decoder. - */ +/** Consumer which skip (throw away) data from the decoder. */ public class SkipConsumer implements Consumer { private final SkipFunction skipFunction; @@ -39,12 +35,10 @@ public void consume(Decoder decoder) throws IOException { } @Override - public void addNull() { - } + public void addNull() {} @Override - public void setPosition(int index) { - } + public void setPosition(int index) {} @Override public FieldVector getVector() { @@ -52,8 +46,7 @@ public FieldVector getVector() { } @Override - public void close() throws Exception { - } + public void close() throws Exception {} @Override public boolean resetValueVector(FieldVector vector) { diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/SkipFunction.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/SkipFunction.java index 93fc4a7fede3f..3d72d03104f3c 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/SkipFunction.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/SkipFunction.java @@ -14,16 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro.consumers; import java.io.IOException; - import org.apache.avro.io.Decoder; -/** - * Adapter function to skip (throw away) data from the decoder. - */ +/** Adapter function to skip (throw away) data from the decoder. */ @FunctionalInterface public interface SkipFunction { void apply(Decoder decoder) throws IOException; diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDateConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDateConsumer.java index a5c36d88fb76a..0f557297a3cb7 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDateConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDateConsumer.java @@ -14,24 +14,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro.consumers.logical; import java.io.IOException; - import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; import org.apache.arrow.vector.DateDayVector; import org.apache.avro.io.Decoder; /** - * Consumer which consume date type values from avro decoder. - * Write the data to {@link DateDayVector}. + * Consumer which consume date type values from avro decoder. Write the data to {@link + * DateDayVector}. */ public class AvroDateConsumer extends BaseAvroConsumer { - /** - * Instantiate a AvroDateConsumer. - */ + /** Instantiate a AvroDateConsumer. */ public AvroDateConsumer(DateDayVector vector) { super(vector); } diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDecimalConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDecimalConsumer.java index ebe5ca3884e5e..fa1a12ac8a6ed 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDecimalConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroDecimalConsumer.java @@ -14,40 +14,32 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro.consumers.logical; import java.io.IOException; import java.nio.ByteBuffer; - import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.DecimalVector; import org.apache.avro.io.Decoder; /** - * Consumer which consume decimal type values from avro decoder. - * Write the data to {@link DecimalVector}. + * Consumer which consume decimal type values from avro decoder. Write the data to {@link + * DecimalVector}. */ public abstract class AvroDecimalConsumer extends BaseAvroConsumer { - /** - * Instantiate a AvroDecimalConsumer. - */ + /** Instantiate a AvroDecimalConsumer. */ public AvroDecimalConsumer(DecimalVector vector) { super(vector); } - /** - * Consumer for decimal logical type with original bytes type. - */ + /** Consumer for decimal logical type with original bytes type. */ public static class BytesDecimalConsumer extends AvroDecimalConsumer { private ByteBuffer cacheBuffer; - /** - * Instantiate a BytesDecimalConsumer. - */ + /** Instantiate a BytesDecimalConsumer. */ public BytesDecimalConsumer(DecimalVector vector) { super(vector); } @@ -60,19 +52,14 @@ public void consume(Decoder decoder) throws IOException { cacheBuffer.get(bytes); vector.setBigEndian(currentIndex++, bytes); } - } - /** - * Consumer for decimal logical type with original fixed type. - */ + /** Consumer for decimal logical type with original fixed type. */ public static class FixedDecimalConsumer extends AvroDecimalConsumer { private byte[] reuseBytes; - /** - * Instantiate a FixedDecimalConsumer. - */ + /** Instantiate a FixedDecimalConsumer. */ public FixedDecimalConsumer(DecimalVector vector, int size) { super(vector); Preconditions.checkArgument(size <= 16, "Decimal bytes length should <= 16."); diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimeMicroConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimeMicroConsumer.java index 89216d4ad1436..60e7d15bf16d6 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimeMicroConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimeMicroConsumer.java @@ -14,24 +14,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro.consumers.logical; import java.io.IOException; - import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; import org.apache.arrow.vector.TimeMicroVector; import org.apache.avro.io.Decoder; /** - * Consumer which consume date time-micro values from avro decoder. - * Write the data to {@link TimeMicroVector}. + * Consumer which consume date time-micro values from avro decoder. Write the data to {@link + * TimeMicroVector}. */ public class AvroTimeMicroConsumer extends BaseAvroConsumer { - /** - * Instantiate a AvroTimeMicroConsumer. - */ + /** Instantiate a AvroTimeMicroConsumer. */ public AvroTimeMicroConsumer(TimeMicroVector vector) { super(vector); } diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimeMillisConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimeMillisConsumer.java index ab5df8d4bc8ac..e0b232e9abd5e 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimeMillisConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimeMillisConsumer.java @@ -14,24 +14,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro.consumers.logical; import java.io.IOException; - import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; import org.apache.arrow.vector.TimeMilliVector; import org.apache.avro.io.Decoder; /** - * Consumer which consume date time-millis values from avro decoder. - * Write the data to {@link TimeMilliVector}. + * Consumer which consume date time-millis values from avro decoder. Write the data to {@link + * TimeMilliVector}. */ public class AvroTimeMillisConsumer extends BaseAvroConsumer { - /** - * Instantiate a AvroTimeMilliConsumer. - */ + /** Instantiate a AvroTimeMilliConsumer. */ public AvroTimeMillisConsumer(TimeMilliVector vector) { super(vector); } diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMicrosConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMicrosConsumer.java index 93b39d479ff0e..88acf7b329569 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMicrosConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMicrosConsumer.java @@ -14,24 +14,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro.consumers.logical; import java.io.IOException; - import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; import org.apache.arrow.vector.TimeStampMicroVector; import org.apache.avro.io.Decoder; /** - * Consumer which consume date timestamp-micro values from avro decoder. - * Write the data to {@link TimeStampMicroVector}. + * Consumer which consume date timestamp-micro values from avro decoder. Write the data to {@link + * TimeStampMicroVector}. */ public class AvroTimestampMicrosConsumer extends BaseAvroConsumer { - /** - * Instantiate a AvroTimestampMicroConsumer. - */ + /** Instantiate a AvroTimestampMicroConsumer. */ public AvroTimestampMicrosConsumer(TimeStampMicroVector vector) { super(vector); } diff --git a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMillisConsumer.java b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMillisConsumer.java index 9e651c3959f81..ec50d7902319c 100644 --- a/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMillisConsumer.java +++ b/java/adapter/avro/src/main/java/org/apache/arrow/adapter/avro/consumers/logical/AvroTimestampMillisConsumer.java @@ -14,24 +14,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro.consumers.logical; import java.io.IOException; - import org.apache.arrow.adapter.avro.consumers.BaseAvroConsumer; import org.apache.arrow.vector.TimeStampMilliVector; import org.apache.avro.io.Decoder; /** - * Consumer which consume date timestamp-millis values from avro decoder. - * Write the data to {@link TimeStampMilliVector}. + * Consumer which consume date timestamp-millis values from avro decoder. Write the data to {@link + * TimeStampMilliVector}. */ public class AvroTimestampMillisConsumer extends BaseAvroConsumer { - /** - * Instantiate a AvroTimestampMillisConsumer. - */ + /** Instantiate a AvroTimestampMillisConsumer. */ public AvroTimestampMillisConsumer(TimeStampMilliVector vector) { super(vector); } diff --git a/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroLogicalTypesTest.java b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroLogicalTypesTest.java index 6ee04e33a5ce1..d8eefc715f611 100644 --- a/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroLogicalTypesTest.java +++ b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroLogicalTypesTest.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro; import static junit.framework.TestCase.assertNull; @@ -27,7 +26,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; - import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.util.DateUtility; @@ -43,13 +41,13 @@ public void testTimestampMicros() throws Exception { Schema schema = getSchema("logical/test_timestamp_micros.avsc"); List data = Arrays.asList(10000L, 20000L, 30000L, 40000L, 50000L); - List expected = Arrays.asList( - DateUtility.getLocalDateTimeFromEpochMicro(10000), - DateUtility.getLocalDateTimeFromEpochMicro(20000), - DateUtility.getLocalDateTimeFromEpochMicro(30000), - DateUtility.getLocalDateTimeFromEpochMicro(40000), - DateUtility.getLocalDateTimeFromEpochMicro(50000) - ); + List expected = + Arrays.asList( + DateUtility.getLocalDateTimeFromEpochMicro(10000), + DateUtility.getLocalDateTimeFromEpochMicro(20000), + DateUtility.getLocalDateTimeFromEpochMicro(30000), + DateUtility.getLocalDateTimeFromEpochMicro(40000), + DateUtility.getLocalDateTimeFromEpochMicro(50000)); VectorSchemaRoot root = writeAndRead(schema, data); FieldVector vector = root.getFieldVectors().get(0); @@ -62,13 +60,13 @@ public void testTimestampMillis() throws Exception { Schema schema = getSchema("logical/test_timestamp_millis.avsc"); List data = Arrays.asList(10000L, 20000L, 30000L, 40000L, 50000L); - List expected = Arrays.asList( - DateUtility.getLocalDateTimeFromEpochMilli(10000), - DateUtility.getLocalDateTimeFromEpochMilli(20000), - DateUtility.getLocalDateTimeFromEpochMilli(30000), - DateUtility.getLocalDateTimeFromEpochMilli(40000), - DateUtility.getLocalDateTimeFromEpochMilli(50000) - ); + List expected = + Arrays.asList( + DateUtility.getLocalDateTimeFromEpochMilli(10000), + DateUtility.getLocalDateTimeFromEpochMilli(20000), + DateUtility.getLocalDateTimeFromEpochMilli(30000), + DateUtility.getLocalDateTimeFromEpochMilli(40000), + DateUtility.getLocalDateTimeFromEpochMilli(50000)); VectorSchemaRoot root = writeAndRead(schema, data); FieldVector vector = root.getFieldVectors().get(0); @@ -93,13 +91,13 @@ public void testTimeMillis() throws Exception { Schema schema = getSchema("logical/test_time_millis.avsc"); List data = Arrays.asList(100, 200, 300, 400, 500); - List expected = Arrays.asList( - DateUtility.getLocalDateTimeFromEpochMilli(100), - DateUtility.getLocalDateTimeFromEpochMilli(200), - DateUtility.getLocalDateTimeFromEpochMilli(300), - DateUtility.getLocalDateTimeFromEpochMilli(400), - DateUtility.getLocalDateTimeFromEpochMilli(500) - ); + List expected = + Arrays.asList( + DateUtility.getLocalDateTimeFromEpochMilli(100), + DateUtility.getLocalDateTimeFromEpochMilli(200), + DateUtility.getLocalDateTimeFromEpochMilli(300), + DateUtility.getLocalDateTimeFromEpochMilli(400), + DateUtility.getLocalDateTimeFromEpochMilli(500)); VectorSchemaRoot root = writeAndRead(schema, data); FieldVector vector = root.getFieldVectors().get(0); @@ -137,7 +135,6 @@ public void testDecimalWithOriginalBytes() throws Exception { VectorSchemaRoot root = writeAndRead(schema, data); FieldVector vector = root.getFieldVectors().get(0); checkPrimitiveResult(expected, vector); - } @Test @@ -174,10 +171,9 @@ public void testInvalidDecimalPrecision() throws Exception { data.add(buffer); } - IllegalArgumentException e = assertThrows(IllegalArgumentException.class, - () -> writeAndRead(schema, data)); + IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> writeAndRead(schema, data)); assertTrue(e.getMessage().contains("Precision must be in range of 1 to 38")); - } @Test @@ -197,5 +193,4 @@ public void testFailedToCreateDecimalLogicalType() throws Exception { Schema schema3 = getSchema("logical/test_decimal_invalid4.avsc"); assertNull(schema3.getLogicalType()); } - } diff --git a/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroSkipFieldTest.java b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroSkipFieldTest.java index 54fa26afe3fa8..3335ee5a8f6dc 100644 --- a/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroSkipFieldTest.java +++ b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroSkipFieldTest.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro; import static org.junit.Assert.assertEquals; @@ -26,7 +25,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Set; - import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.types.Types; @@ -41,7 +39,10 @@ public class AvroSkipFieldTest extends AvroTestBase { public void testSkipUnionWithOneField() throws Exception { Set skipFieldNames = new HashSet<>(); skipFieldNames.add("f0"); - config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + config = + new AvroToArrowConfigBuilder(config.getAllocator()) + .setSkipFieldNames(skipFieldNames) + .build(); Schema schema = getSchema("skip/test_skip_union_before.avsc"); Schema expectedSchema = getSchema("skip/test_skip_union_one_field_expected.avsc"); @@ -70,7 +71,10 @@ public void testSkipUnionWithOneField() throws Exception { public void testSkipUnionWithNullableOneField() throws Exception { Set skipFieldNames = new HashSet<>(); skipFieldNames.add("f1"); - config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + config = + new AvroToArrowConfigBuilder(config.getAllocator()) + .setSkipFieldNames(skipFieldNames) + .build(); Schema schema = getSchema("skip/test_skip_union_before.avsc"); Schema expectedSchema = getSchema("skip/test_skip_union_nullable_field_expected.avsc"); @@ -99,7 +103,10 @@ public void testSkipUnionWithNullableOneField() throws Exception { public void testSkipUnionWithMultiFields() throws Exception { Set skipFieldNames = new HashSet<>(); skipFieldNames.add("f2"); - config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + config = + new AvroToArrowConfigBuilder(config.getAllocator()) + .setSkipFieldNames(skipFieldNames) + .build(); Schema schema = getSchema("skip/test_skip_union_before.avsc"); Schema expectedSchema = getSchema("skip/test_skip_union_multi_fields_expected.avsc"); @@ -128,7 +135,10 @@ public void testSkipUnionWithMultiFields() throws Exception { public void testSkipMapField() throws Exception { Set skipFieldNames = new HashSet<>(); skipFieldNames.add("f1"); - config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + config = + new AvroToArrowConfigBuilder(config.getAllocator()) + .setSkipFieldNames(skipFieldNames) + .build(); Schema schema = getSchema("skip/test_skip_map_before.avsc"); Schema expectedSchema = getSchema("skip/test_skip_map_expected.avsc"); @@ -160,7 +170,10 @@ public void testSkipMapField() throws Exception { public void testSkipArrayField() throws Exception { Set skipFieldNames = new HashSet<>(); skipFieldNames.add("f1"); - config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + config = + new AvroToArrowConfigBuilder(config.getAllocator()) + .setSkipFieldNames(skipFieldNames) + .build(); Schema schema = getSchema("skip/test_skip_array_before.avsc"); Schema expectedSchema = getSchema("skip/test_skip_array_expected.avsc"); @@ -189,7 +202,10 @@ public void testSkipMultiFields() throws Exception { Set skipFieldNames = new HashSet<>(); skipFieldNames.add("f1"); skipFieldNames.add("f2"); - config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + config = + new AvroToArrowConfigBuilder(config.getAllocator()) + .setSkipFieldNames(skipFieldNames) + .build(); Schema schema = getSchema("test_record.avsc"); Schema expectedSchema = getSchema("skip/test_skip_multi_fields_expected.avsc"); @@ -216,7 +232,10 @@ public void testSkipMultiFields() throws Exception { public void testSkipStringField() throws Exception { Set skipFieldNames = new HashSet<>(); skipFieldNames.add("f2"); - config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + config = + new AvroToArrowConfigBuilder(config.getAllocator()) + .setSkipFieldNames(skipFieldNames) + .build(); Schema schema = getSchema("skip/test_skip_base1.avsc"); Schema expectedSchema = getSchema("skip/test_skip_string_expected.avsc"); @@ -229,7 +248,8 @@ public void testSkipStringField() throws Exception { GenericData.Fixed fixed = new GenericData.Fixed(schema.getField("f0").schema()); fixed.bytes(testBytes); record.put(0, fixed); - GenericData.EnumSymbol symbol = new GenericData.EnumSymbol(schema.getField("f1").schema(), "TEST" + i % 2); + GenericData.EnumSymbol symbol = + new GenericData.EnumSymbol(schema.getField("f1").schema(), "TEST" + i % 2); record.put(1, symbol); record.put(2, "testtest" + i); record.put(3, ByteBuffer.wrap(testBytes)); @@ -250,7 +270,10 @@ public void testSkipStringField() throws Exception { public void testSkipBytesField() throws Exception { Set skipFieldNames = new HashSet<>(); skipFieldNames.add("f3"); - config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + config = + new AvroToArrowConfigBuilder(config.getAllocator()) + .setSkipFieldNames(skipFieldNames) + .build(); Schema schema = getSchema("skip/test_skip_base1.avsc"); Schema expectedSchema = getSchema("skip/test_skip_bytes_expected.avsc"); @@ -263,7 +286,8 @@ public void testSkipBytesField() throws Exception { GenericData.Fixed fixed = new GenericData.Fixed(schema.getField("f0").schema()); fixed.bytes(testBytes); record.put(0, fixed); - GenericData.EnumSymbol symbol = new GenericData.EnumSymbol(schema.getField("f1").schema(), "TEST" + i % 2); + GenericData.EnumSymbol symbol = + new GenericData.EnumSymbol(schema.getField("f1").schema(), "TEST" + i % 2); record.put(1, symbol); record.put(2, "testtest" + i); record.put(3, ByteBuffer.wrap(testBytes)); @@ -284,7 +308,10 @@ public void testSkipBytesField() throws Exception { public void testSkipFixedField() throws Exception { Set skipFieldNames = new HashSet<>(); skipFieldNames.add("f0"); - config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + config = + new AvroToArrowConfigBuilder(config.getAllocator()) + .setSkipFieldNames(skipFieldNames) + .build(); Schema schema = getSchema("skip/test_skip_base1.avsc"); Schema expectedSchema = getSchema("skip/test_skip_fixed_expected.avsc"); @@ -297,7 +324,8 @@ public void testSkipFixedField() throws Exception { GenericData.Fixed fixed = new GenericData.Fixed(schema.getField("f0").schema()); fixed.bytes(testBytes); record.put(0, fixed); - GenericData.EnumSymbol symbol = new GenericData.EnumSymbol(schema.getField("f1").schema(), "TEST" + i % 2); + GenericData.EnumSymbol symbol = + new GenericData.EnumSymbol(schema.getField("f1").schema(), "TEST" + i % 2); record.put(1, symbol); record.put(2, "testtest" + i); record.put(3, ByteBuffer.wrap(testBytes)); @@ -318,7 +346,10 @@ public void testSkipFixedField() throws Exception { public void testSkipEnumField() throws Exception { Set skipFieldNames = new HashSet<>(); skipFieldNames.add("f1"); - config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + config = + new AvroToArrowConfigBuilder(config.getAllocator()) + .setSkipFieldNames(skipFieldNames) + .build(); Schema schema = getSchema("skip/test_skip_base1.avsc"); Schema expectedSchema = getSchema("skip/test_skip_fixed_expected.avsc"); @@ -331,7 +362,8 @@ public void testSkipEnumField() throws Exception { GenericData.Fixed fixed = new GenericData.Fixed(schema.getField("f0").schema()); fixed.bytes(testBytes); record.put(0, fixed); - GenericData.EnumSymbol symbol = new GenericData.EnumSymbol(schema.getField("f1").schema(), "TEST" + i % 2); + GenericData.EnumSymbol symbol = + new GenericData.EnumSymbol(schema.getField("f1").schema(), "TEST" + i % 2); record.put(1, symbol); record.put(2, "testtest" + i); record.put(3, ByteBuffer.wrap(testBytes)); @@ -352,7 +384,10 @@ public void testSkipEnumField() throws Exception { public void testSkipBooleanField() throws Exception { Set skipFieldNames = new HashSet<>(); skipFieldNames.add("f0"); - config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + config = + new AvroToArrowConfigBuilder(config.getAllocator()) + .setSkipFieldNames(skipFieldNames) + .build(); Schema schema = getSchema("skip/test_skip_base2.avsc"); Schema expectedSchema = getSchema("skip/test_skip_boolean_expected.avsc"); @@ -385,7 +420,10 @@ public void testSkipBooleanField() throws Exception { public void testSkipIntField() throws Exception { Set skipFieldNames = new HashSet<>(); skipFieldNames.add("f1"); - config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + config = + new AvroToArrowConfigBuilder(config.getAllocator()) + .setSkipFieldNames(skipFieldNames) + .build(); Schema schema = getSchema("skip/test_skip_base2.avsc"); Schema expectedSchema = getSchema("skip/test_skip_int_expected.avsc"); @@ -418,7 +456,10 @@ public void testSkipIntField() throws Exception { public void testSkipLongField() throws Exception { Set skipFieldNames = new HashSet<>(); skipFieldNames.add("f2"); - config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + config = + new AvroToArrowConfigBuilder(config.getAllocator()) + .setSkipFieldNames(skipFieldNames) + .build(); Schema schema = getSchema("skip/test_skip_base2.avsc"); Schema expectedSchema = getSchema("skip/test_skip_long_expected.avsc"); @@ -451,7 +492,10 @@ public void testSkipLongField() throws Exception { public void testSkipFloatField() throws Exception { Set skipFieldNames = new HashSet<>(); skipFieldNames.add("f3"); - config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + config = + new AvroToArrowConfigBuilder(config.getAllocator()) + .setSkipFieldNames(skipFieldNames) + .build(); Schema schema = getSchema("skip/test_skip_base2.avsc"); Schema expectedSchema = getSchema("skip/test_skip_float_expected.avsc"); @@ -484,7 +528,10 @@ public void testSkipFloatField() throws Exception { public void testSkipDoubleField() throws Exception { Set skipFieldNames = new HashSet<>(); skipFieldNames.add("f4"); - config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + config = + new AvroToArrowConfigBuilder(config.getAllocator()) + .setSkipFieldNames(skipFieldNames) + .build(); Schema schema = getSchema("skip/test_skip_base2.avsc"); Schema expectedSchema = getSchema("skip/test_skip_double_expected.avsc"); @@ -517,7 +564,10 @@ public void testSkipDoubleField() throws Exception { public void testSkipRecordField() throws Exception { Set skipFieldNames = new HashSet<>(); skipFieldNames.add("f0"); - config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + config = + new AvroToArrowConfigBuilder(config.getAllocator()) + .setSkipFieldNames(skipFieldNames) + .build(); Schema schema = getSchema("skip/test_skip_record_before.avsc"); Schema nestedSchema = schema.getFields().get(0).schema(); ArrayList data = new ArrayList<>(); @@ -547,7 +597,10 @@ public void testSkipRecordField() throws Exception { public void testSkipNestedFields() throws Exception { Set skipFieldNames = new HashSet<>(); skipFieldNames.add("f0.f0"); - config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + config = + new AvroToArrowConfigBuilder(config.getAllocator()) + .setSkipFieldNames(skipFieldNames) + .build(); Schema schema = getSchema("test_nested_record.avsc"); Schema nestedSchema = schema.getFields().get(0).schema(); ArrayList data = new ArrayList<>(); @@ -603,21 +656,26 @@ public void testSkipThirdLevelField() throws Exception { assertEquals(Types.MinorType.STRUCT, root1.getFieldVectors().get(0).getMinorType()); StructVector secondLevelVector = (StructVector) root1.getFieldVectors().get(0); assertEquals(1, secondLevelVector.getChildrenFromFields().size()); - assertEquals(Types.MinorType.STRUCT, secondLevelVector.getChildrenFromFields().get(0).getMinorType()); + assertEquals( + Types.MinorType.STRUCT, secondLevelVector.getChildrenFromFields().get(0).getMinorType()); StructVector thirdLevelVector = (StructVector) secondLevelVector.getChildrenFromFields().get(0); assertEquals(3, thirdLevelVector.getChildrenFromFields().size()); // skip third level field and validate Set skipFieldNames = new HashSet<>(); skipFieldNames.add("f0.f0.f0"); - config = new AvroToArrowConfigBuilder(config.getAllocator()).setSkipFieldNames(skipFieldNames).build(); + config = + new AvroToArrowConfigBuilder(config.getAllocator()) + .setSkipFieldNames(skipFieldNames) + .build(); VectorSchemaRoot root2 = writeAndRead(firstLevelSchema, data); assertEquals(1, root2.getFieldVectors().size()); assertEquals(Types.MinorType.STRUCT, root2.getFieldVectors().get(0).getMinorType()); StructVector secondStruct = (StructVector) root2.getFieldVectors().get(0); assertEquals(1, secondStruct.getChildrenFromFields().size()); - assertEquals(Types.MinorType.STRUCT, secondStruct.getChildrenFromFields().get(0).getMinorType()); + assertEquals( + Types.MinorType.STRUCT, secondStruct.getChildrenFromFields().get(0).getMinorType()); StructVector thirdStruct = (StructVector) secondStruct.getChildrenFromFields().get(0); assertEquals(2, thirdStruct.getChildrenFromFields().size()); diff --git a/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroTestBase.java b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroTestBase.java index a91bba7b84fb4..534c2cc18c572 100644 --- a/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroTestBase.java +++ b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroTestBase.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro; import static org.junit.Assert.assertEquals; @@ -29,7 +28,6 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; - import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.FieldVector; @@ -51,8 +49,7 @@ public class AvroTestBase { - @ClassRule - public static final TemporaryFolder TMP = new TemporaryFolder(); + @ClassRule public static final TemporaryFolder TMP = new TemporaryFolder(); protected AvroToArrowConfig config; @@ -64,18 +61,21 @@ public void init() { public static Schema getSchema(String schemaName) throws Exception { try { - // Attempt to use JDK 9 behavior of getting the module then the resource stream from the module. + // Attempt to use JDK 9 behavior of getting the module then the resource stream from the + // module. // Note that this code is caller-sensitive. Method getModuleMethod = Class.class.getMethod("getModule"); Object module = getModuleMethod.invoke(TestWriteReadAvroRecord.class); - Method getResourceAsStreamFromModule = module.getClass().getMethod("getResourceAsStream", String.class); - try (InputStream is = (InputStream) getResourceAsStreamFromModule.invoke(module, "/schema/" + schemaName)) { - return new Schema.Parser() - .parse(is); + Method getResourceAsStreamFromModule = + module.getClass().getMethod("getResourceAsStream", String.class); + try (InputStream is = + (InputStream) getResourceAsStreamFromModule.invoke(module, "/schema/" + schemaName)) { + return new Schema.Parser().parse(is); } } catch (NoSuchMethodException ex) { // Use JDK8 behavior. - try (InputStream is = TestWriteReadAvroRecord.class.getResourceAsStream("/schema/" + schemaName)) { + try (InputStream is = + TestWriteReadAvroRecord.class.getResourceAsStream("/schema/" + schemaName)) { return new Schema.Parser().parse(is); } } @@ -84,11 +84,11 @@ public static Schema getSchema(String schemaName) throws Exception { protected VectorSchemaRoot writeAndRead(Schema schema, List data) throws Exception { File dataFile = TMP.newFile(); - BinaryEncoder - encoder = new EncoderFactory().directBinaryEncoder(new FileOutputStream(dataFile), null); + BinaryEncoder encoder = + new EncoderFactory().directBinaryEncoder(new FileOutputStream(dataFile), null); DatumWriter writer = new GenericDatumWriter(schema); - BinaryDecoder - decoder = new DecoderFactory().directBinaryDecoder(new FileInputStream(dataFile), null); + BinaryDecoder decoder = + new DecoderFactory().directBinaryDecoder(new FileInputStream(dataFile), null); for (Object value : data) { writer.write(value, encoder); @@ -157,10 +157,10 @@ protected void checkRecordResult(Schema schema, List data, Vector checkPrimitiveResult(fieldData, root.getFieldVectors().get(i)); } - } - protected void checkNestedRecordResult(Schema schema, List data, VectorSchemaRoot root) { + protected void checkNestedRecordResult( + Schema schema, List data, VectorSchemaRoot root) { assertEquals(data.size(), root.getRowCount()); assertTrue(schema.getFields().size() == 1); @@ -176,10 +176,8 @@ protected void checkNestedRecordResult(Schema schema, List data, checkPrimitiveResult(fieldData, structVector.getChildrenFromFields().get(i)); } - } - // belows are for iterator api protected void checkArrayResult(List> expected, List vectors) { @@ -194,10 +192,12 @@ protected void checkArrayResult(List> expected, List vectors } } - protected void checkRecordResult(Schema schema, List data, List roots) { - roots.forEach(root -> { - assertEquals(schema.getFields().size(), root.getFieldVectors().size()); - }); + protected void checkRecordResult( + Schema schema, List data, List roots) { + roots.forEach( + root -> { + assertEquals(schema.getFields().size(), root.getFieldVectors().size()); + }); for (int i = 0; i < schema.getFields().size(); i++) { List fieldData = new ArrayList(); @@ -210,7 +210,6 @@ protected void checkRecordResult(Schema schema, List data, List vectors) { diff --git a/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowIteratorTest.java b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowIteratorTest.java index 7f2edb08fdabc..7e73b2d6c7038 100644 --- a/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowIteratorTest.java +++ b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowIteratorTest.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro; import static org.junit.Assert.assertEquals; @@ -28,7 +27,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; - import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.util.AutoCloseables; @@ -59,11 +57,11 @@ public void init() { private AvroToArrowVectorIterator convert(Schema schema, List data) throws Exception { File dataFile = TMP.newFile(); - BinaryEncoder - encoder = new EncoderFactory().directBinaryEncoder(new FileOutputStream(dataFile), null); + BinaryEncoder encoder = + new EncoderFactory().directBinaryEncoder(new FileOutputStream(dataFile), null); DatumWriter writer = new GenericDatumWriter(schema); - BinaryDecoder - decoder = new DecoderFactory().directBinaryDecoder(new FileInputStream(dataFile), null); + BinaryDecoder decoder = + new DecoderFactory().directBinaryDecoder(new FileInputStream(dataFile), null); for (Object value : data) { writer.write(value, encoder); @@ -107,7 +105,7 @@ public void testNullableStringType() throws Exception { List roots = new ArrayList<>(); List vectors = new ArrayList<>(); - try (AvroToArrowVectorIterator iterator = convert(schema, data);) { + try (AvroToArrowVectorIterator iterator = convert(schema, data); ) { while (iterator.hasNext()) { VectorSchemaRoot root = iterator.next(); FieldVector vector = root.getFieldVectors().get(0); @@ -117,7 +115,6 @@ public void testNullableStringType() throws Exception { } checkPrimitiveResult(expected, vectors); AutoCloseables.close(roots); - } @Test @@ -140,18 +137,18 @@ public void testRecordType() throws Exception { } checkRecordResult(schema, data, roots); AutoCloseables.close(roots); - } @Test public void testArrayType() throws Exception { Schema schema = getSchema("test_array.avsc"); - List> data = Arrays.asList( - Arrays.asList("11", "222", "999"), - Arrays.asList("12222", "2333", "1000"), - Arrays.asList("1rrr", "2ggg"), - Arrays.asList("1vvv", "2bbb"), - Arrays.asList("1fff", "2")); + List> data = + Arrays.asList( + Arrays.asList("11", "222", "999"), + Arrays.asList("12222", "2333", "1000"), + Arrays.asList("1rrr", "2ggg"), + Arrays.asList("1vvv", "2bbb"), + Arrays.asList("1fff", "2")); List roots = new ArrayList<>(); List vectors = new ArrayList<>(); @@ -172,8 +169,9 @@ public void runLargeNumberOfRows() throws Exception { int x = 0; final int targetRows = 600000; Decoder fakeDecoder = new FakeDecoder(targetRows); - try (AvroToArrowVectorIterator iter = AvroToArrow.avroToArrowIterator(schema, fakeDecoder, - new AvroToArrowConfigBuilder(config.getAllocator()).build())) { + try (AvroToArrowVectorIterator iter = + AvroToArrow.avroToArrowIterator( + schema, fakeDecoder, new AvroToArrowConfigBuilder(config.getAllocator()).build())) { while (iter.hasNext()) { VectorSchemaRoot root = iter.next(); x += root.getRowCount(); @@ -184,9 +182,7 @@ public void runLargeNumberOfRows() throws Exception { assertEquals(targetRows, x); } - /** - * Fake avro decoder to test large data. - */ + /** Fake avro decoder to test large data. */ private static class FakeDecoder extends Decoder { private int numRows; @@ -204,8 +200,7 @@ private void validate() throws EOFException { } @Override - public void readNull() throws IOException { - } + public void readNull() throws IOException {} @Override public boolean readBoolean() throws IOException { @@ -243,9 +238,7 @@ public String readString() throws IOException { } @Override - public void skipString() throws IOException { - - } + public void skipString() throws IOException {} @Override public ByteBuffer readBytes(ByteBuffer old) throws IOException { @@ -253,9 +246,7 @@ public ByteBuffer readBytes(ByteBuffer old) throws IOException { } @Override - public void skipBytes() throws IOException { - - } + public void skipBytes() throws IOException {} @Override public void readFixed(byte[] bytes, int start, int length) throws IOException { @@ -264,9 +255,7 @@ public void readFixed(byte[] bytes, int start, int length) throws IOException { } @Override - public void skipFixed(int length) throws IOException { - - } + public void skipFixed(int length) throws IOException {} @Override public int readEnum() throws IOException { diff --git a/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowTest.java b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowTest.java index 26f72173b6b7e..59317c3be033f 100644 --- a/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowTest.java +++ b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/AvroToArrowTest.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro; import static org.junit.Assert.assertEquals; @@ -26,7 +25,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; - import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.VarCharVector; import org.apache.arrow.vector.VectorSchemaRoot; @@ -105,12 +103,13 @@ public void testFixedAttributes() throws Exception { @Test public void testEnumAttributes() throws Exception { Schema schema = getSchema("attrs/test_enum_attrs.avsc"); - List data = Arrays.asList( - new GenericData.EnumSymbol(schema, "SPADES"), - new GenericData.EnumSymbol(schema, "HEARTS"), - new GenericData.EnumSymbol(schema, "DIAMONDS"), - new GenericData.EnumSymbol(schema, "CLUBS"), - new GenericData.EnumSymbol(schema, "SPADES")); + List data = + Arrays.asList( + new GenericData.EnumSymbol(schema, "SPADES"), + new GenericData.EnumSymbol(schema, "HEARTS"), + new GenericData.EnumSymbol(schema, "DIAMONDS"), + new GenericData.EnumSymbol(schema, "CLUBS"), + new GenericData.EnumSymbol(schema, "SPADES")); VectorSchemaRoot root = writeAndRead(schema, data); FieldVector vector = root.getFieldVectors().get(0); @@ -172,12 +171,13 @@ public void testNestedRecordType() throws Exception { @Test public void testEnumType() throws Exception { Schema schema = getSchema("test_primitive_enum.avsc"); - List data = Arrays.asList( - new GenericData.EnumSymbol(schema, "SPADES"), - new GenericData.EnumSymbol(schema, "HEARTS"), - new GenericData.EnumSymbol(schema, "DIAMONDS"), - new GenericData.EnumSymbol(schema, "CLUBS"), - new GenericData.EnumSymbol(schema, "SPADES")); + List data = + Arrays.asList( + new GenericData.EnumSymbol(schema, "SPADES"), + new GenericData.EnumSymbol(schema, "HEARTS"), + new GenericData.EnumSymbol(schema, "DIAMONDS"), + new GenericData.EnumSymbol(schema, "CLUBS"), + new GenericData.EnumSymbol(schema, "SPADES")); List expectedIndices = Arrays.asList(0, 1, 2, 3, 0); @@ -302,12 +302,13 @@ public void testNullableDoubleType() throws Exception { @Test public void testBytesType() throws Exception { Schema schema = getSchema("test_primitive_bytes.avsc"); - List data = Arrays.asList( - ByteBuffer.wrap("value1".getBytes(StandardCharsets.UTF_8)), - ByteBuffer.wrap("value2".getBytes(StandardCharsets.UTF_8)), - ByteBuffer.wrap("value3".getBytes(StandardCharsets.UTF_8)), - ByteBuffer.wrap("value4".getBytes(StandardCharsets.UTF_8)), - ByteBuffer.wrap("value5".getBytes(StandardCharsets.UTF_8))); + List data = + Arrays.asList( + ByteBuffer.wrap("value1".getBytes(StandardCharsets.UTF_8)), + ByteBuffer.wrap("value2".getBytes(StandardCharsets.UTF_8)), + ByteBuffer.wrap("value3".getBytes(StandardCharsets.UTF_8)), + ByteBuffer.wrap("value4".getBytes(StandardCharsets.UTF_8)), + ByteBuffer.wrap("value5".getBytes(StandardCharsets.UTF_8))); VectorSchemaRoot root = writeAndRead(schema, data); FieldVector vector = root.getFieldVectors().get(0); @@ -322,7 +323,8 @@ public void testNullableBytesType() throws Exception { ArrayList data = new ArrayList<>(); for (int i = 0; i < 5; i++) { GenericRecord record = new GenericData.Record(schema); - record.put(0, i % 2 == 0 ? ByteBuffer.wrap(("test" + i).getBytes(StandardCharsets.UTF_8)) : null); + record.put( + 0, i % 2 == 0 ? ByteBuffer.wrap(("test" + i).getBytes(StandardCharsets.UTF_8)) : null); data.add(record); } @@ -359,12 +361,13 @@ public void testNullableBooleanType() throws Exception { @Test public void testArrayType() throws Exception { Schema schema = getSchema("test_array.avsc"); - List> data = Arrays.asList( - Arrays.asList("11", "222", "999"), - Arrays.asList("12222", "2333", "1000"), - Arrays.asList("1rrr", "2ggg"), - Arrays.asList("1vvv", "2bbb"), - Arrays.asList("1fff", "2")); + List> data = + Arrays.asList( + Arrays.asList("11", "222", "999"), + Arrays.asList("12222", "2333", "1000"), + Arrays.asList("1rrr", "2ggg"), + Arrays.asList("1vvv", "2bbb"), + Arrays.asList("1fff", "2")); VectorSchemaRoot root = writeAndRead(schema, data); FieldVector vector = root.getFieldVectors().get(0); @@ -471,5 +474,4 @@ public void testNullableUnionType() throws Exception { checkPrimitiveResult(expected, vector); } - } diff --git a/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/TestWriteReadAvroRecord.java b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/TestWriteReadAvroRecord.java index afbddaa6ed87a..a721a1e4cc6a8 100644 --- a/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/TestWriteReadAvroRecord.java +++ b/java/adapter/avro/src/test/java/org/apache/arrow/adapter/avro/TestWriteReadAvroRecord.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.avro; import static org.junit.Assert.assertEquals; @@ -22,7 +21,6 @@ import java.io.File; import java.util.ArrayList; import java.util.List; - import org.apache.avro.Schema; import org.apache.avro.file.DataFileReader; import org.apache.avro.file.DataFileWriter; @@ -36,11 +34,9 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; - public class TestWriteReadAvroRecord { - @ClassRule - public static final TemporaryFolder TMP = new TemporaryFolder(); + @ClassRule public static final TemporaryFolder TMP = new TemporaryFolder(); @Test public void testWriteAndRead() throws Exception { @@ -48,7 +44,7 @@ public void testWriteAndRead() throws Exception { File dataFile = TMP.newFile(); Schema schema = AvroTestBase.getSchema("test.avsc"); - //write data to disk + // write data to disk GenericRecord user1 = new GenericData.Record(schema); user1.put("name", "Alyssa"); user1.put("favorite_number", 256); @@ -65,10 +61,10 @@ public void testWriteAndRead() throws Exception { dataFileWriter.append(user2); dataFileWriter.close(); - //read data from disk + // read data from disk DatumReader datumReader = new GenericDatumReader(schema); - DataFileReader - dataFileReader = new DataFileReader(dataFile, datumReader); + DataFileReader dataFileReader = + new DataFileReader(dataFile, datumReader); List result = new ArrayList<>(); while (dataFileReader.hasNext()) { GenericRecord user = dataFileReader.next(); @@ -86,5 +82,4 @@ public void testWriteAndRead() throws Exception { assertEquals(7, deUser2.get("favorite_number")); assertEquals("red", deUser2.get("favorite_color").toString()); } - } diff --git a/java/adapter/jdbc/pom.xml b/java/adapter/jdbc/pom.xml index 2f2911dd9da95..b444eff56277d 100644 --- a/java/adapter/jdbc/pom.xml +++ b/java/adapter/jdbc/pom.xml @@ -24,6 +24,11 @@ (Contrib/Experimental)A library for converting JDBC data to Arrow data. http://maven.apache.org + + dev/checkstyle/checkstyle-spotless.xml + none + + diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowVectorIterator.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowVectorIterator.java index 427c766982f30..d30cf32a04996 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowVectorIterator.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowVectorIterator.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc; import static org.apache.arrow.adapter.jdbc.JdbcToArrowUtils.isColumnNullable; @@ -23,7 +22,6 @@ import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.util.Iterator; - import org.apache.arrow.adapter.jdbc.consumer.CompositeJdbcConsumer; import org.apache.arrow.adapter.jdbc.consumer.JdbcConsumer; import org.apache.arrow.adapter.jdbc.consumer.exceptions.JdbcConsumerException; @@ -35,9 +33,7 @@ import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.ValueVectorUtility; -/** - * VectorSchemaRoot iterator for partially converting JDBC data. - */ +/** VectorSchemaRoot iterator for partially converting JDBC data. */ public class ArrowVectorIterator implements Iterator, AutoCloseable { private final ResultSet resultSet; @@ -54,13 +50,12 @@ public class ArrowVectorIterator implements Iterator, AutoClos private final int targetBatchSize; - // This is used to track whether the ResultSet has been fully read, and is needed specifically for cases where there + // This is used to track whether the ResultSet has been fully read, and is needed specifically for + // cases where there // is a ResultSet having zero rows (empty): private boolean readComplete = false; - /** - * Construct an instance. - */ + /** Construct an instance. */ private ArrowVectorIterator(ResultSet resultSet, JdbcToArrowConfig config) throws SQLException { this.resultSet = resultSet; this.config = config; @@ -73,12 +68,8 @@ private ArrowVectorIterator(ResultSet resultSet, JdbcToArrowConfig config) throw this.nextBatch = config.isReuseVectorSchemaRoot() ? createVectorSchemaRoot() : null; } - /** - * Create a ArrowVectorIterator to partially convert data. - */ - public static ArrowVectorIterator create( - ResultSet resultSet, - JdbcToArrowConfig config) + /** Create a ArrowVectorIterator to partially convert data. */ + public static ArrowVectorIterator create(ResultSet resultSet, JdbcToArrowConfig config) throws SQLException { ArrowVectorIterator iterator = null; try { @@ -142,10 +133,18 @@ private VectorSchemaRoot createVectorSchemaRoot() throws SQLException { private void initialize(VectorSchemaRoot root) throws SQLException { for (int i = 1; i <= consumers.length; i++) { - final JdbcFieldInfo columnFieldInfo = JdbcToArrowUtils.getJdbcFieldInfoForColumn(rsmd, i, config); + final JdbcFieldInfo columnFieldInfo = + JdbcToArrowUtils.getJdbcFieldInfoForColumn(rsmd, i, config); ArrowType arrowType = config.getJdbcToArrowTypeConverter().apply(columnFieldInfo); - consumers[i - 1] = config.getJdbcConsumerGetter().apply( - arrowType, i, isColumnNullable(resultSet.getMetaData(), i, columnFieldInfo), root.getVector(i - 1), config); + consumers[i - 1] = + config + .getJdbcConsumerGetter() + .apply( + arrowType, + i, + isColumnNullable(resultSet.getMetaData(), i, columnFieldInfo), + root.getVector(i - 1), + config); } } @@ -170,16 +169,17 @@ public boolean hasNext() { } /** - * Gets the next vector. - * If {@link JdbcToArrowConfig#isReuseVectorSchemaRoot()} is false, - * the client is responsible for freeing its resources. + * Gets the next vector. If {@link JdbcToArrowConfig#isReuseVectorSchemaRoot()} is false, the + * client is responsible for freeing its resources. + * * @throws JdbcConsumerException on error from VectorConsumer */ @Override public VectorSchemaRoot next() { Preconditions.checkArgument(hasNext()); try { - VectorSchemaRoot ret = config.isReuseVectorSchemaRoot() ? nextBatch : createVectorSchemaRoot(); + VectorSchemaRoot ret = + config.isReuseVectorSchemaRoot() ? nextBatch : createVectorSchemaRoot(); load(ret); return ret; } catch (Exception e) { @@ -193,8 +193,9 @@ public VectorSchemaRoot next() { } /** - * Clean up resources ONLY WHEN THE {@link VectorSchemaRoot} HOLDING EACH BATCH IS REUSED. If a new VectorSchemaRoot - * is created for each batch, each root must be closed manually by the client code. + * Clean up resources ONLY WHEN THE {@link VectorSchemaRoot} HOLDING EACH BATCH IS REUSED. If a + * new VectorSchemaRoot is created for each batch, each root must be closed manually by the client + * code. */ @Override public void close() { diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/Constants.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/Constants.java index f95133fc7e44c..30e734a68d511 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/Constants.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/Constants.java @@ -14,20 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc; -/** - * String constants used for metadata returned on Vectors. - */ +/** String constants used for metadata returned on Vectors. */ public class Constants { - private Constants() { - } + private Constants() {} public static final String SQL_CATALOG_NAME_KEY = "SQL_CATALOG_NAME"; public static final String SQL_SCHEMA_NAME_KEY = "SQL_SCHEMA_NAME"; public static final String SQL_TABLE_NAME_KEY = "SQL_TABLE_NAME"; public static final String SQL_COLUMN_NAME_KEY = "SQL_COLUMN_NAME"; public static final String SQL_TYPE_KEY = "SQL_TYPE"; - } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcFieldInfo.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcFieldInfo.java index d16964ea14417..6becac0bbc10c 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcFieldInfo.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcFieldInfo.java @@ -14,25 +14,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Types; - import org.apache.arrow.util.Preconditions; /** - * This class represents the information about a JDBC ResultSet Field that is - * needed to construct an {@link org.apache.arrow.vector.types.pojo.ArrowType}. - * Currently, this is: + * This class represents the information about a JDBC ResultSet Field that is needed to construct an + * {@link org.apache.arrow.vector.types.pojo.ArrowType}. Currently, this is: + * *
      - *
    • The JDBC {@link java.sql.Types} type.
      • - *
    • The nullability.
      • - *
    • The field's precision (used for {@link java.sql.Types#DECIMAL} and {@link java.sql.Types#NUMERIC} types).
      • - *
    • The field's scale (used for {@link java.sql.Types#DECIMAL} and {@link java.sql.Types#NUMERIC} types).
      • + *
    • The JDBC {@link java.sql.Types} type. + *
    • The nullability. + *
    • The field's precision (used for {@link java.sql.Types#DECIMAL} and {@link + * java.sql.Types#NUMERIC} types). + *
    • The field's scale (used for {@link java.sql.Types#DECIMAL} and {@link + * java.sql.Types#NUMERIC} types). *
    */ public class JdbcFieldInfo { @@ -45,12 +45,13 @@ public class JdbcFieldInfo { private final int displaySize; /** - * Builds a JdbcFieldInfo using only the {@link java.sql.Types} type. Do not use this constructor - * if the field type is {@link java.sql.Types#DECIMAL} or {@link java.sql.Types#NUMERIC}; the precision and - * scale will be set to 0. + * Builds a JdbcFieldInfo using only the {@link java.sql.Types} type. Do not use this + * constructor if the field type is {@link java.sql.Types#DECIMAL} or {@link + * java.sql.Types#NUMERIC}; the precision and scale will be set to 0. * * @param jdbcType The {@link java.sql.Types} type. - * @throws IllegalArgumentException if jdbcType is {@link java.sql.Types#DECIMAL} or {@link java.sql.Types#NUMERIC}. + * @throws IllegalArgumentException if jdbcType is {@link java.sql.Types#DECIMAL} or {@link + * java.sql.Types#NUMERIC}. */ public JdbcFieldInfo(int jdbcType) { Preconditions.checkArgument( @@ -67,7 +68,8 @@ public JdbcFieldInfo(int jdbcType) { /** * Builds a JdbcFieldInfo from the {@link java.sql.Types} type, precision, and scale. - * Use this constructor for {@link java.sql.Types#DECIMAL} and {@link java.sql.Types#NUMERIC} types. + * Use this constructor for {@link java.sql.Types#DECIMAL} and {@link java.sql.Types#NUMERIC} + * types. * * @param jdbcType The {@link java.sql.Types} type. * @param precision The field's numeric precision. @@ -84,11 +86,13 @@ public JdbcFieldInfo(int jdbcType, int precision, int scale) { } /** - * Builds a JdbcFieldInfo from the {@link java.sql.Types} type, nullability, precision, and scale. + * Builds a JdbcFieldInfo from the {@link java.sql.Types} type, nullability, + * precision, and scale. * * @param jdbcType The {@link java.sql.Types} type. * @param nullability The nullability. Must be one of {@link ResultSetMetaData#columnNoNulls}, - * {@link ResultSetMetaData#columnNullable}, or {@link ResultSetMetaData#columnNullableUnknown}. + * {@link ResultSetMetaData#columnNullable}, or {@link + * ResultSetMetaData#columnNullableUnknown}. * @param precision The field's numeric precision. * @param scale The field's numeric scale. */ @@ -103,7 +107,8 @@ public JdbcFieldInfo(int jdbcType, int nullability, int precision, int scale) { } /** - * Builds a JdbcFieldInfo from the corresponding {@link java.sql.ResultSetMetaData} column. + * Builds a JdbcFieldInfo from the corresponding {@link java.sql.ResultSetMetaData} + * column. * * @param rsmd The {@link java.sql.ResultSetMetaData} to get the field information from. * @param column The column to get the field information for (on a 1-based index). @@ -113,10 +118,12 @@ public JdbcFieldInfo(int jdbcType, int nullability, int precision, int scale) { */ public JdbcFieldInfo(ResultSetMetaData rsmd, int column) throws SQLException { Preconditions.checkNotNull(rsmd, "ResultSetMetaData cannot be null."); - Preconditions.checkArgument(column > 0, "ResultSetMetaData columns have indices starting at 1."); + Preconditions.checkArgument( + column > 0, "ResultSetMetaData columns have indices starting at 1."); Preconditions.checkArgument( column <= rsmd.getColumnCount(), - "The index must be within the number of columns (1 to %s, inclusive)", rsmd.getColumnCount()); + "The index must be within the number of columns (1 to %s, inclusive)", + rsmd.getColumnCount()); this.column = column; this.jdbcType = rsmd.getColumnType(column); @@ -128,8 +135,8 @@ public JdbcFieldInfo(ResultSetMetaData rsmd, int column) throws SQLException { } /** - * Builds a JdbcFieldInfo from the corresponding row from a {@link java.sql.DatabaseMetaData#getColumns} - * ResultSet. + * Builds a JdbcFieldInfo from the corresponding row from a {@link + * java.sql.DatabaseMetaData#getColumns} ResultSet. * * @param rs The {@link java.sql.ResultSet} to get the field information from. * @throws SQLException If the column information cannot be retrieved. @@ -144,51 +151,42 @@ public JdbcFieldInfo(ResultSet rs) throws SQLException { this.displaySize = rs.getInt("CHAR_OCTET_LENGTH"); } - /** - * The {@link java.sql.Types} type. - */ + /** The {@link java.sql.Types} type. */ public int getJdbcType() { return jdbcType; } - /** - * The nullability. - */ + /** The nullability. */ public int isNullable() { return nullability; } /** - * The numeric precision, for {@link java.sql.Types#NUMERIC} and {@link java.sql.Types#DECIMAL} types. + * The numeric precision, for {@link java.sql.Types#NUMERIC} and {@link java.sql.Types#DECIMAL} + * types. */ public int getPrecision() { return precision; } /** - * The numeric scale, for {@link java.sql.Types#NUMERIC} and {@link java.sql.Types#DECIMAL} types. + * The numeric scale, for {@link java.sql.Types#NUMERIC} and {@link java.sql.Types#DECIMAL} types. */ public int getScale() { return scale; } - /** - * The column index for query column. - */ + /** The column index for query column. */ public int getColumn() { return column; } - /** - * The type name as reported by the database. - */ + /** The type name as reported by the database. */ public String getTypeName() { return typeName; } - /** - * The max number of characters for the column. - */ + /** The max number of characters for the column. */ public int getDisplaySize() { return displaySize; } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcParameterBinder.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcParameterBinder.java index 2dfc0658cb8d1..fd4721bcd9c4e 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcParameterBinder.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcParameterBinder.java @@ -14,14 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc; import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.HashMap; import java.util.Map; - import org.apache.arrow.adapter.jdbc.binder.ColumnBinder; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.VectorSchemaRoot; @@ -29,8 +27,8 @@ /** * A binder binds JDBC prepared statement parameters to rows of Arrow data from a VectorSchemaRoot. * - * Each row of the VectorSchemaRoot will be bound to the configured parameters of the PreparedStatement. - * One row of data is bound at a time. + *

    Each row of the VectorSchemaRoot will be bound to the configured parameters of the + * PreparedStatement. One row of data is bound at a time. */ public class JdbcParameterBinder { private final PreparedStatement statement; @@ -44,8 +42,10 @@ public class JdbcParameterBinder { * * @param statement The statement to bind parameters to. * @param root The VectorSchemaRoot to pull data from. - * @param binders Column binders to translate from Arrow data to JDBC parameters, one per parameter. - * @param parameterIndices For each binder in binders, the index of the parameter to bind to. + * @param binders Column binders to translate from Arrow data to JDBC parameters, one per + * parameter. + * @param parameterIndices For each binder in binders, the index of the parameter to bind + * to. */ private JdbcParameterBinder( final PreparedStatement statement, @@ -55,7 +55,8 @@ private JdbcParameterBinder( Preconditions.checkArgument( binders.length == parameterIndices.length, "Number of column binders (%s) must equal number of parameter indices (%s)", - binders.length, parameterIndices.length); + binders.length, + parameterIndices.length); this.statement = statement; this.root = root; this.binders = binders; @@ -66,9 +67,10 @@ private JdbcParameterBinder( /** * Initialize a binder with a builder. * - * @param statement The statement to bind to. The binder does not maintain ownership of the statement. - * @param root The {@link VectorSchemaRoot} to pull data from. The binder does not maintain ownership - * of the vector schema root. + * @param statement The statement to bind to. The binder does not maintain ownership of the + * statement. + * @param root The {@link VectorSchemaRoot} to pull data from. The binder does not maintain + * ownership of the vector schema root. */ public static Builder builder(final PreparedStatement statement, final VectorSchemaRoot root) { return new Builder(statement, root); @@ -82,8 +84,8 @@ public void reset() { /** * Bind the next row of data to the parameters of the statement. * - * After this, the application should call the desired method on the prepared statement, - * such as {@link PreparedStatement#executeUpdate()}, or {@link PreparedStatement#addBatch()}. + *

    After this, the application should call the desired method on the prepared statement, such + * as {@link PreparedStatement#executeUpdate()}, or {@link PreparedStatement#addBatch()}. * * @return true if a row was bound, false if rows were exhausted */ @@ -99,9 +101,7 @@ public boolean next() throws SQLException { return true; } - /** - * A builder for a {@link JdbcParameterBinder}. - */ + /** A builder for a {@link JdbcParameterBinder}. */ public static class Builder { private final PreparedStatement statement; private final VectorSchemaRoot root; @@ -123,9 +123,7 @@ public Builder bindAll() { /** Bind the given parameter to the given column using the default binder. */ public Builder bind(int parameterIndex, int columnIndex) { - return bind( - parameterIndex, - ColumnBinder.forVector(root.getVector(columnIndex))); + return bind(parameterIndex, ColumnBinder.forVector(root.getVector(columnIndex))); } /** Bind the given parameter using the given binder. */ diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java index 246451b5b22f9..493e53056f945 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java @@ -14,13 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc; import java.io.IOException; import java.sql.ResultSet; import java.sql.SQLException; - import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.util.Preconditions; @@ -29,44 +27,32 @@ * *

    This utility uses following data mapping to map JDBC/SQL datatype to Arrow data types. * - *

    CHAR --> ArrowType.Utf8 - * NCHAR --> ArrowType.Utf8 - * VARCHAR --> ArrowType.Utf8 - * NVARCHAR --> ArrowType.Utf8 - * LONGVARCHAR --> ArrowType.Utf8 - * LONGNVARCHAR --> ArrowType.Utf8 - * NUMERIC --> ArrowType.Decimal(precision, scale) - * DECIMAL --> ArrowType.Decimal(precision, scale) - * BIT --> ArrowType.Bool - * TINYINT --> ArrowType.Int(8, signed) - * SMALLINT --> ArrowType.Int(16, signed) - * INTEGER --> ArrowType.Int(32, signed) - * BIGINT --> ArrowType.Int(64, signed) - * REAL --> ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE) - * FLOAT --> ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE) - * DOUBLE --> ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE) - * BINARY --> ArrowType.Binary - * VARBINARY --> ArrowType.Binary - * LONGVARBINARY --> ArrowType.Binary - * DATE --> ArrowType.Date(DateUnit.MILLISECOND) - * TIME --> ArrowType.Time(TimeUnit.MILLISECOND, 32) - * TIMESTAMP --> ArrowType.Timestamp(TimeUnit.MILLISECOND, timezone=null) - * CLOB --> ArrowType.Utf8 - * BLOB --> ArrowType.Binary + *

    CHAR --> ArrowType.Utf8 NCHAR --> ArrowType.Utf8 VARCHAR --> ArrowType.Utf8 NVARCHAR --> + * ArrowType.Utf8 LONGVARCHAR --> ArrowType.Utf8 LONGNVARCHAR --> ArrowType.Utf8 NUMERIC --> + * ArrowType.Decimal(precision, scale) DECIMAL --> ArrowType.Decimal(precision, scale) BIT --> + * ArrowType.Bool TINYINT --> ArrowType.Int(8, signed) SMALLINT --> ArrowType.Int(16, signed) + * INTEGER --> ArrowType.Int(32, signed) BIGINT --> ArrowType.Int(64, signed) REAL --> + * ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE) FLOAT --> + * ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE) DOUBLE --> + * ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE) BINARY --> ArrowType.Binary VARBINARY --> + * ArrowType.Binary LONGVARBINARY --> ArrowType.Binary DATE --> ArrowType.Date(DateUnit.MILLISECOND) + * TIME --> ArrowType.Time(TimeUnit.MILLISECOND, 32) TIMESTAMP --> + * ArrowType.Timestamp(TimeUnit.MILLISECOND, timezone=null) CLOB --> ArrowType.Utf8 BLOB --> + * ArrowType.Binary * * @since 0.10.0 */ public class JdbcToArrow { /*----------------------------------------------------------------* - | | - | Partial Convert API | - | | - *----------------------------------------------------------------*/ + | | + | Partial Convert API | + | | + *----------------------------------------------------------------*/ /** - * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. - * Note here uses the default targetBatchSize = 1024. + * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow + * objects. Note here uses the default targetBatchSize = 1024. * * @param resultSet ResultSet to use to fetch the data from underlying database * @param allocator Memory allocator @@ -74,28 +60,25 @@ public class JdbcToArrow { * @throws SQLException on error */ public static ArrowVectorIterator sqlToArrowVectorIterator( - ResultSet resultSet, - BufferAllocator allocator) - throws SQLException, IOException { + ResultSet resultSet, BufferAllocator allocator) throws SQLException, IOException { Preconditions.checkNotNull(allocator, "Memory Allocator object cannot be null"); - JdbcToArrowConfig config = - new JdbcToArrowConfig(allocator, JdbcToArrowUtils.getUtcCalendar()); + JdbcToArrowConfig config = new JdbcToArrowConfig(allocator, JdbcToArrowUtils.getUtcCalendar()); return sqlToArrowVectorIterator(resultSet, config); } /** - * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. - * Note if not specify {@link JdbcToArrowConfig#targetBatchSize}, will use default value 1024. + * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow + * objects. Note if not specify {@link JdbcToArrowConfig#targetBatchSize}, will use default value + * 1024. + * * @param resultSet ResultSet to use to fetch the data from underlying database - * @param config Configuration of the conversion from JDBC to Arrow. + * @param config Configuration of the conversion from JDBC to Arrow. * @return Arrow Data Objects {@link ArrowVectorIterator} * @throws SQLException on error */ public static ArrowVectorIterator sqlToArrowVectorIterator( - ResultSet resultSet, - JdbcToArrowConfig config) - throws SQLException, IOException { + ResultSet resultSet, JdbcToArrowConfig config) throws SQLException, IOException { Preconditions.checkNotNull(resultSet, "JDBC ResultSet object cannot be null"); Preconditions.checkNotNull(config, "The configuration cannot be null"); return ArrowVectorIterator.create(resultSet, config); diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfig.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfig.java index 68851f4a98bc9..1bfcfc8fe00aa 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfig.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfig.java @@ -14,14 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc; import java.math.RoundingMode; import java.util.Calendar; import java.util.Map; import java.util.function.Function; - import org.apache.arrow.adapter.jdbc.consumer.JdbcConsumer; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.util.Preconditions; @@ -30,25 +28,23 @@ /** * This class configures the JDBC-to-Arrow conversion process. - *

    - * The allocator is used to construct the {@link org.apache.arrow.vector.VectorSchemaRoot}, - * and the calendar is used to define the time zone of any - * {@link org.apache.arrow.vector.types.pojo.ArrowType.Timestamp} - * fields that are created during the conversion. Neither field may be null. - *

    - *

    - * If the includeMetadata flag is set, the Arrow field metadata will contain information - * from the corresponding {@link java.sql.ResultSetMetaData} that was used to create the - * {@link org.apache.arrow.vector.types.pojo.FieldType} of the corresponding - * {@link org.apache.arrow.vector.FieldVector}. - *

    - *

    - * If there are any {@link java.sql.Types#ARRAY} fields in the {@link java.sql.ResultSet}, the corresponding - * {@link JdbcFieldInfo} for the array's contents must be defined here. Unfortunately, the sub-type - * information cannot be retrieved from all JDBC implementations (H2 for example, returns - * {@link java.sql.Types#NULL} for the array sub-type), so it must be configured here. The column index - * or name can be used to map to a {@link JdbcFieldInfo}, and that will be used for the conversion. - *

    + * + *

    The allocator is used to construct the {@link org.apache.arrow.vector.VectorSchemaRoot}, and + * the calendar is used to define the time zone of any {@link + * org.apache.arrow.vector.types.pojo.ArrowType.Timestamp} fields that are created during the + * conversion. Neither field may be null. + * + *

    If the includeMetadata flag is set, the Arrow field metadata will contain + * information from the corresponding {@link java.sql.ResultSetMetaData} that was used to create the + * {@link org.apache.arrow.vector.types.pojo.FieldType} of the corresponding {@link + * org.apache.arrow.vector.FieldVector}. + * + *

    If there are any {@link java.sql.Types#ARRAY} fields in the {@link java.sql.ResultSet}, the + * corresponding {@link JdbcFieldInfo} for the array's contents must be defined here. Unfortunately, + * the sub-type information cannot be retrieved from all JDBC implementations (H2 for example, + * returns {@link java.sql.Types#NULL} for the array sub-type), so it must be configured here. The + * column index or name can be used to map to a {@link JdbcFieldInfo}, and that will be used for the + * conversion. */ public final class JdbcToArrowConfig { @@ -66,14 +62,12 @@ public final class JdbcToArrowConfig { private final Map> columnMetadataByColumnIndex; private final RoundingMode bigDecimalRoundingMode; /** - * The maximum rowCount to read each time when partially convert data. - * Default value is 1024 and -1 means disable partial read. - * default is -1 which means disable partial read. - * Note that this flag only useful for {@link JdbcToArrow#sqlToArrowVectorIterator} - * 1) if targetBatchSize != -1, it will convert full data into multiple vectors - * with valueCount no more than targetBatchSize. - * 2) if targetBatchSize == -1, it will convert full data into a single vector in {@link ArrowVectorIterator} - *

    + * The maximum rowCount to read each time when partially convert data. Default value is 1024 and + * -1 means disable partial read. default is -1 which means disable partial read. Note that this + * flag only useful for {@link JdbcToArrow#sqlToArrowVectorIterator} 1) if targetBatchSize != -1, + * it will convert full data into multiple vectors with valueCount no more than targetBatchSize. + * 2) if targetBatchSize == -1, it will convert full data into a single vector in {@link + * ArrowVectorIterator} */ private final int targetBatchSize; @@ -81,81 +75,100 @@ public final class JdbcToArrowConfig { private final JdbcConsumerFactory jdbcConsumerGetter; /** - * Constructs a new configuration from the provided allocator and calendar. The allocator - * is used when constructing the Arrow vectors from the ResultSet, and the calendar is used to define - * Arrow Timestamp fields, and to read time-based fields from the JDBC ResultSet. + * Constructs a new configuration from the provided allocator and calendar. The allocator + * is used when constructing the Arrow vectors from the ResultSet, and the calendar is + * used to define Arrow Timestamp fields, and to read time-based fields from the JDBC + * ResultSet. * - * @param allocator The memory allocator to construct the Arrow vectors with. - * @param calendar The calendar to use when constructing Timestamp fields and reading time-based results. + * @param allocator The memory allocator to construct the Arrow vectors with. + * @param calendar The calendar to use when constructing Timestamp fields and reading time-based + * results. */ JdbcToArrowConfig(BufferAllocator allocator, Calendar calendar) { - this(allocator, calendar, + this( + allocator, + calendar, /* include metadata */ false, /* reuse vector schema root */ false, /* array sub-types by column index */ null, /* array sub-types by column name */ null, - DEFAULT_TARGET_BATCH_SIZE, null, null); + DEFAULT_TARGET_BATCH_SIZE, + null, + null); } JdbcToArrowConfig( - BufferAllocator allocator, - Calendar calendar, - boolean includeMetadata, - boolean reuseVectorSchemaRoot, - Map arraySubTypesByColumnIndex, - Map arraySubTypesByColumnName, - int targetBatchSize, - Function jdbcToArrowTypeConverter) { - this(allocator, calendar, includeMetadata, reuseVectorSchemaRoot, arraySubTypesByColumnIndex, - arraySubTypesByColumnName, targetBatchSize, jdbcToArrowTypeConverter, null); + BufferAllocator allocator, + Calendar calendar, + boolean includeMetadata, + boolean reuseVectorSchemaRoot, + Map arraySubTypesByColumnIndex, + Map arraySubTypesByColumnName, + int targetBatchSize, + Function jdbcToArrowTypeConverter) { + this( + allocator, + calendar, + includeMetadata, + reuseVectorSchemaRoot, + arraySubTypesByColumnIndex, + arraySubTypesByColumnName, + targetBatchSize, + jdbcToArrowTypeConverter, + null); } /** - * Constructs a new configuration from the provided allocator and calendar. The allocator - * is used when constructing the Arrow vectors from the ResultSet, and the calendar is used to define - * Arrow Timestamp fields, and to read time-based fields from the JDBC ResultSet. + * Constructs a new configuration from the provided allocator and calendar. The allocator + * is used when constructing the Arrow vectors from the ResultSet, and the calendar is + * used to define Arrow Timestamp fields, and to read time-based fields from the JDBC + * ResultSet. * - * @param allocator The memory allocator to construct the Arrow vectors with. - * @param calendar The calendar to use when constructing Timestamp fields and reading time-based results. - * @param includeMetadata Whether to include JDBC field metadata in the Arrow Schema Field metadata. + * @param allocator The memory allocator to construct the Arrow vectors with. + * @param calendar The calendar to use when constructing Timestamp fields and reading time-based + * results. + * @param includeMetadata Whether to include JDBC field metadata in the Arrow Schema Field + * metadata. * @param reuseVectorSchemaRoot Whether to reuse the vector schema root for each data load. * @param arraySubTypesByColumnIndex The type of the JDBC array at the column index (1-based). - * @param arraySubTypesByColumnName The type of the JDBC array at the column name. - * @param targetBatchSize The target batch size to be used in preallocation of the resulting vectors. - * @param jdbcToArrowTypeConverter The function that maps JDBC field type information to arrow type. If set to null, - * the default mapping will be used, which is defined as: - *
      - *
    • CHAR --> ArrowType.Utf8
      • - *
    • NCHAR --> ArrowType.Utf8
      • - *
    • VARCHAR --> ArrowType.Utf8
      • - *
    • NVARCHAR --> ArrowType.Utf8
      • - *
    • LONGVARCHAR --> ArrowType.Utf8
      • - *
    • LONGNVARCHAR --> ArrowType.Utf8
      • - *
    • NUMERIC --> ArrowType.Decimal(precision, scale)
      • - *
    • DECIMAL --> ArrowType.Decimal(precision, scale)
      • - *
    • BIT --> ArrowType.Bool
      • - *
    • TINYINT --> ArrowType.Int(8, signed)
      • - *
    • SMALLINT --> ArrowType.Int(16, signed)
      • - *
    • INTEGER --> ArrowType.Int(32, signed)
      • - *
    • BIGINT --> ArrowType.Int(64, signed)
      • - *
    • REAL --> ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)
      • - *
    • FLOAT --> ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)
      • - *
    • DOUBLE --> ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)
      • - *
    • BINARY --> ArrowType.Binary
      • - *
    • VARBINARY --> ArrowType.Binary
      • - *
    • LONGVARBINARY --> ArrowType.Binary
      • - *
    • DATE --> ArrowType.Date(DateUnit.DAY)
      • - *
    • TIME --> ArrowType.Time(TimeUnit.MILLISECOND, 32)
      • - *
    • TIMESTAMP --> ArrowType.Timestamp(TimeUnit.MILLISECOND, calendar timezone)
      • - *
    • CLOB --> ArrowType.Utf8
      • - *
    • BLOB --> ArrowType.Binary
      • - *
    • ARRAY --> ArrowType.List
      • - *
    • STRUCT --> ArrowType.Struct
      • - *
    • NULL --> ArrowType.Null
      • - *
    - * @param bigDecimalRoundingMode The java.math.RoundingMode to be used in coercion of a BigDecimal from a - * ResultSet having a scale which does not match that of the target vector. Use null - * (default value) to require strict scale matching. + * @param arraySubTypesByColumnName The type of the JDBC array at the column name. + * @param targetBatchSize The target batch size to be used in preallocation of the resulting + * vectors. + * @param jdbcToArrowTypeConverter The function that maps JDBC field type information to arrow + * type. If set to null, the default mapping will be used, which is defined as: + *
      + *
    • CHAR --> ArrowType.Utf8 + *
    • NCHAR --> ArrowType.Utf8 + *
    • VARCHAR --> ArrowType.Utf8 + *
    • NVARCHAR --> ArrowType.Utf8 + *
    • LONGVARCHAR --> ArrowType.Utf8 + *
    • LONGNVARCHAR --> ArrowType.Utf8 + *
    • NUMERIC --> ArrowType.Decimal(precision, scale) + *
    • DECIMAL --> ArrowType.Decimal(precision, scale) + *
    • BIT --> ArrowType.Bool + *
    • TINYINT --> ArrowType.Int(8, signed) + *
    • SMALLINT --> ArrowType.Int(16, signed) + *
    • INTEGER --> ArrowType.Int(32, signed) + *
    • BIGINT --> ArrowType.Int(64, signed) + *
    • REAL --> ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE) + *
    • FLOAT --> ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE) + *
    • DOUBLE --> ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE) + *
    • BINARY --> ArrowType.Binary + *
    • VARBINARY --> ArrowType.Binary + *
    • LONGVARBINARY --> ArrowType.Binary + *
    • DATE --> ArrowType.Date(DateUnit.DAY) + *
    • TIME --> ArrowType.Time(TimeUnit.MILLISECOND, 32) + *
    • TIMESTAMP --> ArrowType.Timestamp(TimeUnit.MILLISECOND, calendar timezone) + *
    • CLOB --> ArrowType.Utf8 + *
    • BLOB --> ArrowType.Binary + *
    • ARRAY --> ArrowType.List + *
    • STRUCT --> ArrowType.Struct + *
    • NULL --> ArrowType.Null + *
    + * + * @param bigDecimalRoundingMode The java.math.RoundingMode to be used in coercion of a BigDecimal + * from a ResultSet having a scale which does not match that of the target vector. Use null + * (default value) to require strict scale matching. */ JdbcToArrowConfig( BufferAllocator allocator, @@ -245,16 +258,19 @@ public final class JdbcToArrowConfig { this.bigDecimalRoundingMode = bigDecimalRoundingMode; // set up type converter - this.jdbcToArrowTypeConverter = jdbcToArrowTypeConverter != null ? jdbcToArrowTypeConverter : - (jdbcFieldInfo) -> JdbcToArrowUtils.getArrowTypeFromJdbcType(jdbcFieldInfo, calendar); + this.jdbcToArrowTypeConverter = + jdbcToArrowTypeConverter != null + ? jdbcToArrowTypeConverter + : (jdbcFieldInfo) -> JdbcToArrowUtils.getArrowTypeFromJdbcType(jdbcFieldInfo, calendar); - this.jdbcConsumerGetter = jdbcConsumerGetter != null ? jdbcConsumerGetter : JdbcToArrowUtils::getConsumer; + this.jdbcConsumerGetter = + jdbcConsumerGetter != null ? jdbcConsumerGetter : JdbcToArrowUtils::getConsumer; } /** - * The calendar to use when defining Arrow Timestamp fields - * and retrieving {@link java.sql.Date}, {@link java.sql.Time}, or {@link java.sql.Timestamp} - * data types from the {@link java.sql.ResultSet}, or null if not converting. + * The calendar to use when defining Arrow Timestamp fields and retrieving {@link java.sql.Date}, + * {@link java.sql.Time}, or {@link java.sql.Timestamp} data types from the {@link + * java.sql.ResultSet}, or null if not converting. * * @return the calendar. */ @@ -280,30 +296,22 @@ public boolean shouldIncludeMetadata() { return includeMetadata; } - /** - * Get the target batch size for partial read. - */ + /** Get the target batch size for partial read. */ public int getTargetBatchSize() { return targetBatchSize; } - /** - * Get whether it is allowed to reuse the vector schema root. - */ + /** Get whether it is allowed to reuse the vector schema root. */ public boolean isReuseVectorSchemaRoot() { return reuseVectorSchemaRoot; } - /** - * Gets the mapping between JDBC type information to Arrow type. - */ + /** Gets the mapping between JDBC type information to Arrow type. */ public Function getJdbcToArrowTypeConverter() { return jdbcToArrowTypeConverter; } - /** - * Gets the JDBC consumer getter. - */ + /** Gets the JDBC consumer getter. */ public JdbcConsumerFactory getJdbcConsumerGetter() { return jdbcConsumerGetter; } @@ -311,8 +319,10 @@ public JdbcConsumerFactory getJdbcConsumerGetter() { /** * Returns the array sub-type {@link JdbcFieldInfo} defined for the provided column index. * - * @param index The {@link java.sql.ResultSetMetaData} column index of an {@link java.sql.Types#ARRAY} type. - * @return The {@link JdbcFieldInfo} for that array's sub-type, or null if not defined. + * @param index The {@link java.sql.ResultSetMetaData} column index of an {@link + * java.sql.Types#ARRAY} type. + * @return The {@link JdbcFieldInfo} for that array's sub-type, or null if not + * defined. */ public JdbcFieldInfo getArraySubTypeByColumnIndex(int index) { if (arraySubTypesByColumnIndex == null) { @@ -325,8 +335,10 @@ public JdbcFieldInfo getArraySubTypeByColumnIndex(int index) { /** * Returns the array sub-type {@link JdbcFieldInfo} defined for the provided column name. * - * @param name The {@link java.sql.ResultSetMetaData} column name of an {@link java.sql.Types#ARRAY} type. - * @return The {@link JdbcFieldInfo} for that array's sub-type, or null if not defined. + * @param name The {@link java.sql.ResultSetMetaData} column name of an {@link + * java.sql.Types#ARRAY} type. + * @return The {@link JdbcFieldInfo} for that array's sub-type, or null if not + * defined. */ public JdbcFieldInfo getArraySubTypeByColumnName(String name) { if (arraySubTypesByColumnName == null) { @@ -339,7 +351,8 @@ public JdbcFieldInfo getArraySubTypeByColumnName(String name) { /** * Returns the type {@link JdbcFieldInfo} explicitly defined for the provided column index. * - * @param index The {@link java.sql.ResultSetMetaData} column index to evaluate for explicit type mapping. + * @param index The {@link java.sql.ResultSetMetaData} column index to evaluate for explicit type + * mapping. * @return The {@link JdbcFieldInfo} defined for the column, or null if not defined. */ public JdbcFieldInfo getExplicitTypeByColumnIndex(int index) { @@ -353,7 +366,8 @@ public JdbcFieldInfo getExplicitTypeByColumnIndex(int index) { /** * Returns the type {@link JdbcFieldInfo} explicitly defined for the provided column name. * - * @param name The {@link java.sql.ResultSetMetaData} column name to evaluate for explicit type mapping. + * @param name The {@link java.sql.ResultSetMetaData} column name to evaluate for explicit type + * mapping. * @return The {@link JdbcFieldInfo} defined for the column, or null if not defined. */ public JdbcFieldInfo getExplicitTypeByColumnName(String name) { @@ -364,17 +378,12 @@ public JdbcFieldInfo getExplicitTypeByColumnName(String name) { } } - /** - * Return schema level metadata or null if not provided. - */ + /** Return schema level metadata or null if not provided. */ public Map getSchemaMetadata() { return schemaMetadata; } - /** - * Return metadata from columnIndex->meta map on per field basis - * or null if not provided. - */ + /** Return metadata from columnIndex->meta map on per field basis or null if not provided. */ public Map> getColumnMetadataByColumnIndex() { return columnMetadataByColumnIndex; } @@ -383,12 +392,14 @@ public RoundingMode getBigDecimalRoundingMode() { return bigDecimalRoundingMode; } - /** - * Interface for a function that gets a JDBC consumer for the given values. - */ + /** Interface for a function that gets a JDBC consumer for the given values. */ @FunctionalInterface public interface JdbcConsumerFactory { - JdbcConsumer apply(ArrowType arrowType, int columnIndex, boolean nullable, FieldVector vector, - JdbcToArrowConfig config); + JdbcConsumer apply( + ArrowType arrowType, + int columnIndex, + boolean nullable, + FieldVector vector, + JdbcToArrowConfig config); } } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigBuilder.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigBuilder.java index 7d88c23832067..783a373c6d0a7 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigBuilder.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigBuilder.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc; import static org.apache.arrow.adapter.jdbc.JdbcToArrowConfig.DEFAULT_TARGET_BATCH_SIZE; @@ -23,15 +22,12 @@ import java.util.Calendar; import java.util.Map; import java.util.function.Function; - import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.types.pojo.ArrowType; -/** - * This class builds {@link JdbcToArrowConfig}s. - */ +/** This class builds {@link JdbcToArrowConfig}s. */ public class JdbcToArrowConfigBuilder { private Calendar calendar; private BufferAllocator allocator; @@ -49,9 +45,9 @@ public class JdbcToArrowConfigBuilder { private RoundingMode bigDecimalRoundingMode; /** - * Default constructor for the JdbcToArrowConfigBuilder}. - * Use the setter methods for the allocator and calendar; the allocator must be - * set. Otherwise, {@link #build()} will throw a {@link NullPointerException}. + * Default constructor for the JdbcToArrowConfigBuilder}. Use the setter methods for + * the allocator and calendar; the allocator must be set. Otherwise, {@link #build()} will throw a + * {@link NullPointerException}. */ public JdbcToArrowConfigBuilder() { this.allocator = null; @@ -68,16 +64,13 @@ public JdbcToArrowConfigBuilder() { } /** - * Constructor for the JdbcToArrowConfigBuilder. The - * allocator is required, and a {@link NullPointerException} - * will be thrown if it is null. - *

    - * The allocator is used to construct Arrow vectors from the JDBC ResultSet. - * The calendar is used to determine the time zone of {@link java.sql.Timestamp} - * fields and convert {@link java.sql.Date}, {@link java.sql.Time}, and - * {@link java.sql.Timestamp} fields to a single, common time zone when reading - * from the result set. - *

    + * Constructor for the JdbcToArrowConfigBuilder. The allocator is required, and a + * {@link NullPointerException} will be thrown if it is null. + * + *

    The allocator is used to construct Arrow vectors from the JDBC ResultSet. The calendar is + * used to determine the time zone of {@link java.sql.Timestamp} fields and convert {@link + * java.sql.Date}, {@link java.sql.Time}, and {@link java.sql.Timestamp} fields to a single, + * common time zone when reading from the result set. * * @param allocator The Arrow Vector memory allocator. * @param calendar The calendar to use when constructing timestamp fields. @@ -95,26 +88,23 @@ public JdbcToArrowConfigBuilder(BufferAllocator allocator, Calendar calendar) { } /** - * Constructor for the JdbcToArrowConfigBuilder. Both the - * allocator and calendar are required. A {@link NullPointerException} - * will be thrown if either of those arguments is null. - *

    - * The allocator is used to construct Arrow vectors from the JDBC ResultSet. - * The calendar is used to determine the time zone of {@link java.sql.Timestamp} - * fields and convert {@link java.sql.Date}, {@link java.sql.Time}, and - * {@link java.sql.Timestamp} fields to a single, common time zone when reading - * from the result set. - *

    - *

    - * The includeMetadata argument, if true will cause - * various information about each database field to be added to the Vector - * Schema's field metadata. - *

    + * Constructor for the JdbcToArrowConfigBuilder. Both the allocator and calendar are + * required. A {@link NullPointerException} will be thrown if either of those arguments is + * null. + * + *

    The allocator is used to construct Arrow vectors from the JDBC ResultSet. The calendar is + * used to determine the time zone of {@link java.sql.Timestamp} fields and convert {@link + * java.sql.Date}, {@link java.sql.Time}, and {@link java.sql.Timestamp} fields to a single, + * common time zone when reading from the result set. + * + *

    The includeMetadata argument, if true will cause various + * information about each database field to be added to the Vector Schema's field metadata. * * @param allocator The Arrow Vector memory allocator. * @param calendar The calendar to use when constructing timestamp fields. */ - public JdbcToArrowConfigBuilder(BufferAllocator allocator, Calendar calendar, boolean includeMetadata) { + public JdbcToArrowConfigBuilder( + BufferAllocator allocator, Calendar calendar, boolean includeMetadata) { this(allocator, calendar); this.includeMetadata = includeMetadata; } @@ -132,8 +122,8 @@ public JdbcToArrowConfigBuilder setAllocator(BufferAllocator allocator) { } /** - * Sets the {@link Calendar} to use when constructing timestamp fields in the - * Arrow schema, and reading time-based fields from the JDBC ResultSet. + * Sets the {@link Calendar} to use when constructing timestamp fields in the Arrow schema, and + * reading time-based fields from the JDBC ResultSet. * * @param calendar the calendar to set. */ @@ -145,7 +135,8 @@ public JdbcToArrowConfigBuilder setCalendar(Calendar calendar) { /** * Sets whether to include JDBC ResultSet field metadata in the Arrow Schema field metadata. * - * @param includeMetadata Whether to include or exclude JDBC metadata in the Arrow Schema field metadata. + * @param includeMetadata Whether to include or exclude JDBC metadata in the Arrow Schema field + * metadata. * @return This instance of the JdbcToArrowConfig, for chaining. */ public JdbcToArrowConfigBuilder setIncludeMetadata(boolean includeMetadata) { @@ -154,8 +145,8 @@ public JdbcToArrowConfigBuilder setIncludeMetadata(boolean includeMetadata) { } /** - * Sets the mapping of column-index-to-{@link JdbcFieldInfo} used for columns of type {@link java.sql.Types#ARRAY}. - * The column index is 1-based, to match the JDBC column index. + * Sets the mapping of column-index-to-{@link JdbcFieldInfo} used for columns of type {@link + * java.sql.Types#ARRAY}. The column index is 1-based, to match the JDBC column index. * * @param map The mapping. * @return This instance of the JdbcToArrowConfig, for chaining. @@ -166,7 +157,8 @@ public JdbcToArrowConfigBuilder setArraySubTypeByColumnIndexMap(MapJdbcToArrowConfig, for chaining. @@ -178,11 +170,12 @@ public JdbcToArrowConfigBuilder setArraySubTypeByColumnNameMap(Map - * This can be useful to override type information from JDBC drivers that provide incomplete type info, - * e.g. DECIMAL with precision = scale = 0. - *

    - * The column index is 1-based, to match the JDBC column index. + * + *

    This can be useful to override type information from JDBC drivers that provide incomplete + * type info, e.g. DECIMAL with precision = scale = 0. + * + *

    The column index is 1-based, to match the JDBC column index. + * * @param map The mapping. */ public JdbcToArrowConfigBuilder setExplicitTypesByColumnIndex(Map map) { @@ -192,9 +185,10 @@ public JdbcToArrowConfigBuilder setExplicitTypesByColumnIndex(Map - * This can be useful to override type information from JDBC drivers that provide incomplete type info, - * e.g. DECIMAL with precision = scale = 0. + * + *

    This can be useful to override type information from JDBC drivers that provide incomplete + * type info, e.g. DECIMAL with precision = scale = 0. + * * @param map The mapping. */ public JdbcToArrowConfigBuilder setExplicitTypesByColumnName(Map map) { @@ -204,8 +198,8 @@ public JdbcToArrowConfigBuilder setExplicitTypesByColumnName(Map - * Use {@link JdbcToArrowConfig#NO_LIMIT_BATCH_SIZE} to read all rows at once. + * + *

    Use {@link JdbcToArrowConfig#NO_LIMIT_BATCH_SIZE} to read all rows at once. */ public JdbcToArrowConfigBuilder setTargetBatchSize(int targetBatchSize) { this.targetBatchSize = targetBatchSize; @@ -214,8 +208,9 @@ public JdbcToArrowConfigBuilder setTargetBatchSize(int targetBatchSize) { /** * Set the function used to convert JDBC types to Arrow types. - *

    - * Defaults to wrapping {@link JdbcToArrowUtils#getArrowTypeFromJdbcType(JdbcFieldInfo, Calendar)}. + * + *

    Defaults to wrapping {@link JdbcToArrowUtils#getArrowTypeFromJdbcType(JdbcFieldInfo, + * Calendar)}. */ public JdbcToArrowConfigBuilder setJdbcToArrowTypeConverter( Function jdbcToArrowTypeConverter) { @@ -225,9 +220,9 @@ public JdbcToArrowConfigBuilder setJdbcToArrowTypeConverter( /** * Set the function used to get a JDBC consumer for a given type. - *

    - * Defaults to wrapping {@link - * JdbcToArrowUtils#getConsumer(ArrowType, Integer, Boolean, FieldVector, JdbcToArrowConfig)}. + * + *

    Defaults to wrapping {@link JdbcToArrowUtils#getConsumer(ArrowType, Integer, Boolean, + * FieldVector, JdbcToArrowConfig)}. */ public JdbcToArrowConfigBuilder setJdbcConsumerGetter( JdbcToArrowConfig.JdbcConsumerFactory jdbcConsumerGetter) { @@ -236,35 +231,32 @@ public JdbcToArrowConfigBuilder setJdbcConsumerGetter( } /** - * Set whether to use the same {@link org.apache.arrow.vector.VectorSchemaRoot} instance on each iteration, - * or to allocate a new one. + * Set whether to use the same {@link org.apache.arrow.vector.VectorSchemaRoot} instance on each + * iteration, or to allocate a new one. */ public JdbcToArrowConfigBuilder setReuseVectorSchemaRoot(boolean reuseVectorSchemaRoot) { this.reuseVectorSchemaRoot = reuseVectorSchemaRoot; return this; } - /** - * Set metadata for schema. - */ + /** Set metadata for schema. */ public JdbcToArrowConfigBuilder setSchemaMetadata(Map schemaMetadata) { this.schemaMetadata = schemaMetadata; return this; } - /** - * Set metadata from columnIndex->meta map on per field basis. - */ + /** Set metadata from columnIndex->meta map on per field basis. */ public JdbcToArrowConfigBuilder setColumnMetadataByColumnIndex( - Map> columnMetadataByColumnIndex) { + Map> columnMetadataByColumnIndex) { this.columnMetadataByColumnIndex = columnMetadataByColumnIndex; return this; } /** - * Set the rounding mode used when the scale of the actual value does not match the declared scale. - *

    - * By default, an error is raised in such cases. + * Set the rounding mode used when the scale of the actual value does not match the declared + * scale. + * + *

    By default, an error is raised in such cases. */ public JdbcToArrowConfigBuilder setBigDecimalRoundingMode(RoundingMode bigDecimalRoundingMode) { this.bigDecimalRoundingMode = bigDecimalRoundingMode; @@ -272,8 +264,8 @@ public JdbcToArrowConfigBuilder setBigDecimalRoundingMode(RoundingMode bigDecima } /** - * This builds the {@link JdbcToArrowConfig} from the provided - * {@link BufferAllocator} and {@link Calendar}. + * This builds the {@link JdbcToArrowConfig} from the provided {@link BufferAllocator} and {@link + * Calendar}. * * @return The built {@link JdbcToArrowConfig} * @throws NullPointerException if either the allocator or calendar was not set. diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java index eaee49936079f..8397d4c9e0dc4 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrowUtils.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc; import static org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE; @@ -38,7 +37,6 @@ import java.util.Locale; import java.util.Map; import java.util.TimeZone; - import org.apache.arrow.adapter.jdbc.consumer.ArrayConsumer; import org.apache.arrow.adapter.jdbc.consumer.BigIntConsumer; import org.apache.arrow.adapter.jdbc.consumer.BinaryConsumer; @@ -91,7 +89,8 @@ import org.apache.arrow.vector.util.ValueVectorUtility; /** - * Class that does most of the work to convert JDBC ResultSet data into Arrow columnar format Vector objects. + * Class that does most of the work to convert JDBC ResultSet data into Arrow columnar format Vector + * objects. * * @since 0.10.0 */ @@ -99,9 +98,7 @@ public class JdbcToArrowUtils { private static final int JDBC_ARRAY_VALUE_COLUMN = 2; - /** - * Returns the instance of a {java.util.Calendar} with the UTC time zone and root locale. - */ + /** Returns the instance of a {java.util.Calendar} with the UTC time zone and root locale. */ public static Calendar getUtcCalendar() { return Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT); } @@ -114,7 +111,8 @@ public static Calendar getUtcCalendar() { * @return {@link Schema} * @throws SQLException on error */ - public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd, Calendar calendar) throws SQLException { + public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd, Calendar calendar) + throws SQLException { Preconditions.checkNotNull(calendar, "Calendar object can't be null"); return jdbcToArrowSchema(rsmd, new JdbcToArrowConfig(new RootAllocator(0), calendar)); @@ -123,25 +121,28 @@ public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd, Calendar calendar /** * Create Arrow {@link Schema} object for the given JDBC {@link ResultSetMetaData}. * - * @param parameterMetaData The ResultSetMetaData containing the results, to read the JDBC metadata from. - * @param calendar The calendar to use the time zone field of, to construct Timestamp fields from. + * @param parameterMetaData The ResultSetMetaData containing the results, to read the JDBC + * metadata from. + * @param calendar The calendar to use the time zone field of, to construct Timestamp fields from. * @return {@link Schema} * @throws SQLException on error */ - public static Schema jdbcToArrowSchema(final ParameterMetaData parameterMetaData, final Calendar calendar) - throws SQLException { + public static Schema jdbcToArrowSchema( + final ParameterMetaData parameterMetaData, final Calendar calendar) throws SQLException { Preconditions.checkNotNull(calendar, "Calendar object can't be null"); Preconditions.checkNotNull(parameterMetaData); final List parameterFields = new ArrayList<>(parameterMetaData.getParameterCount()); - for (int parameterCounter = 1; parameterCounter <= parameterMetaData.getParameterCount(); - parameterCounter++) { + for (int parameterCounter = 1; + parameterCounter <= parameterMetaData.getParameterCount(); + parameterCounter++) { final int jdbcDataType = parameterMetaData.getParameterType(parameterCounter); final int jdbcIsNullable = parameterMetaData.isNullable(parameterCounter); final boolean arrowIsNullable = jdbcIsNullable != ParameterMetaData.parameterNoNulls; final int precision = parameterMetaData.getPrecision(parameterCounter); final int scale = parameterMetaData.getScale(parameterCounter); - final ArrowType arrowType = getArrowTypeFromJdbcType(new JdbcFieldInfo(jdbcDataType, precision, scale), calendar); - final FieldType fieldType = new FieldType(arrowIsNullable, arrowType, /*dictionary=*/null); + final ArrowType arrowType = + getArrowTypeFromJdbcType(new JdbcFieldInfo(jdbcDataType, precision, scale), calendar); + final FieldType fieldType = new FieldType(arrowIsNullable, arrowType, /*dictionary=*/ null); parameterFields.add(new Field(null, fieldType, null)); } @@ -152,10 +153,11 @@ public static Schema jdbcToArrowSchema(final ParameterMetaData parameterMetaData * Converts the provided JDBC type to its respective {@link ArrowType} counterpart. * * @param fieldInfo the {@link JdbcFieldInfo} with information about the original JDBC type. - * @param calendar the {@link Calendar} to use for datetime data types. + * @param calendar the {@link Calendar} to use for datetime data types. * @return a new {@link ArrowType}. */ - public static ArrowType getArrowTypeFromJdbcType(final JdbcFieldInfo fieldInfo, final Calendar calendar) { + public static ArrowType getArrowTypeFromJdbcType( + final JdbcFieldInfo fieldInfo, final Calendar calendar) { switch (fieldInfo.getJdbcType()) { case Types.BOOLEAN: case Types.BIT: @@ -222,30 +224,34 @@ public static ArrowType getArrowTypeFromJdbcType(final JdbcFieldInfo fieldInfo, /** * Create Arrow {@link Schema} object for the given JDBC {@link java.sql.ResultSetMetaData}. * - *

    - * If {@link JdbcToArrowConfig#shouldIncludeMetadata()} returns true, the following fields - * will be added to the {@link FieldType#getMetadata()}: + *

    If {@link JdbcToArrowConfig#shouldIncludeMetadata()} returns true, the + * following fields will be added to the {@link FieldType#getMetadata()}: + * *

      - *
    • {@link Constants#SQL_CATALOG_NAME_KEY} representing {@link ResultSetMetaData#getCatalogName(int)}
      • - *
    • {@link Constants#SQL_TABLE_NAME_KEY} representing {@link ResultSetMetaData#getTableName(int)}
      • - *
    • {@link Constants#SQL_COLUMN_NAME_KEY} representing {@link ResultSetMetaData#getColumnLabel(int)}
      • - *
    • {@link Constants#SQL_TYPE_KEY} representing {@link ResultSetMetaData#getColumnTypeName(int)}
      • + *
    • {@link Constants#SQL_CATALOG_NAME_KEY} representing {@link + * ResultSetMetaData#getCatalogName(int)} + *
    • {@link Constants#SQL_TABLE_NAME_KEY} representing {@link + * ResultSetMetaData#getTableName(int)} + *
    • {@link Constants#SQL_COLUMN_NAME_KEY} representing {@link + * ResultSetMetaData#getColumnLabel(int)} + *
    • {@link Constants#SQL_TYPE_KEY} representing {@link + * ResultSetMetaData#getColumnTypeName(int)} *
    - *

    - *

    - * If any columns are of type {@link java.sql.Types#ARRAY}, the configuration object will be used to look up - * the array sub-type field. The {@link JdbcToArrowConfig#getArraySubTypeByColumnIndex(int)} method will be - * checked first, followed by the {@link JdbcToArrowConfig#getArraySubTypeByColumnName(String)} method. - *

    + * + *

    If any columns are of type {@link java.sql.Types#ARRAY}, the configuration object will be + * used to look up the array sub-type field. The {@link + * JdbcToArrowConfig#getArraySubTypeByColumnIndex(int)} method will be checked first, followed by + * the {@link JdbcToArrowConfig#getArraySubTypeByColumnName(String)} method. * * @param rsmd The ResultSetMetaData containing the results, to read the JDBC metadata from. * @param config The configuration to use when constructing the schema. * @return {@link Schema} * @throws SQLException on error - * @throws IllegalArgumentException if rsmd contains an {@link java.sql.Types#ARRAY} but the - * config does not have a sub-type definition for it. + * @throws IllegalArgumentException if rsmd contains an {@link java.sql.Types#ARRAY} + * but the config does not have a sub-type definition for it. */ - public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd, JdbcToArrowConfig config) throws SQLException { + public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd, JdbcToArrowConfig config) + throws SQLException { Preconditions.checkNotNull(rsmd, "JDBC ResultSetMetaData object can't be null"); Preconditions.checkNotNull(config, "The configuration object must not be null"); @@ -254,8 +260,10 @@ public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd, JdbcToArrowConfig for (int i = 1; i <= columnCount; i++) { final String columnName = rsmd.getColumnLabel(i); - final Map columnMetadata = config.getColumnMetadataByColumnIndex() != null ? - config.getColumnMetadataByColumnIndex().get(i) : null; + final Map columnMetadata = + config.getColumnMetadataByColumnIndex() != null + ? config.getColumnMetadataByColumnIndex().get(i) + : null; final Map metadata; if (config.shouldIncludeMetadata()) { metadata = new HashMap<>(); @@ -278,14 +286,19 @@ public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd, JdbcToArrowConfig final JdbcFieldInfo columnFieldInfo = getJdbcFieldInfoForColumn(rsmd, i, config); final ArrowType arrowType = config.getJdbcToArrowTypeConverter().apply(columnFieldInfo); if (arrowType != null) { - final FieldType fieldType = new FieldType( - isColumnNullable(rsmd, i, columnFieldInfo), arrowType, /* dictionary encoding */ null, metadata); + final FieldType fieldType = + new FieldType( + isColumnNullable(rsmd, i, columnFieldInfo), + arrowType, /* dictionary encoding */ + null, + metadata); List children = null; if (arrowType.getTypeID() == ArrowType.List.TYPE_TYPE) { final JdbcFieldInfo arrayFieldInfo = getJdbcFieldInfoForArraySubType(rsmd, i, config); if (arrayFieldInfo == null) { - throw new IllegalArgumentException("Configuration does not provide a mapping for array column " + i); + throw new IllegalArgumentException( + "Configuration does not provide a mapping for array column " + i); } children = new ArrayList(); final ArrowType childType = config.getJdbcToArrowTypeConverter().apply(arrayFieldInfo); @@ -295,9 +308,13 @@ public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd, JdbcToArrowConfig FieldType keyType = new FieldType(false, new ArrowType.Utf8(), null, null); FieldType valueType = new FieldType(false, new ArrowType.Utf8(), null, null); children = new ArrayList<>(); - children.add(new Field("child", mapType, - Arrays.asList(new Field(MapVector.KEY_NAME, keyType, null), - new Field(MapVector.VALUE_NAME, valueType, null)))); + children.add( + new Field( + "child", + mapType, + Arrays.asList( + new Field(MapVector.KEY_NAME, keyType, null), + new Field(MapVector.VALUE_NAME, valueType, null)))); } fields.add(new Field(columnName, fieldType, children)); @@ -307,18 +324,14 @@ public static Schema jdbcToArrowSchema(ResultSetMetaData rsmd, JdbcToArrowConfig } static JdbcFieldInfo getJdbcFieldInfoForColumn( - ResultSetMetaData rsmd, - int arrayColumn, - JdbcToArrowConfig config) - throws SQLException { + ResultSetMetaData rsmd, int arrayColumn, JdbcToArrowConfig config) throws SQLException { Preconditions.checkNotNull(rsmd, "ResultSet MetaData object cannot be null"); Preconditions.checkNotNull(config, "Configuration must not be null"); Preconditions.checkArgument( - arrayColumn > 0, - "ResultSetMetaData columns start with 1; column cannot be less than 1"); + arrayColumn > 0, "ResultSetMetaData columns start with 1; column cannot be less than 1"); Preconditions.checkArgument( - arrayColumn <= rsmd.getColumnCount(), - "Column number cannot be more than the number of columns"); + arrayColumn <= rsmd.getColumnCount(), + "Column number cannot be more than the number of columns"); JdbcFieldInfo fieldInfo = config.getExplicitTypeByColumnIndex(arrayColumn); if (fieldInfo == null) { @@ -334,16 +347,12 @@ static JdbcFieldInfo getJdbcFieldInfoForColumn( * If no sub-type can be found, returns null. */ private static JdbcFieldInfo getJdbcFieldInfoForArraySubType( - ResultSetMetaData rsmd, - int arrayColumn, - JdbcToArrowConfig config) - throws SQLException { + ResultSetMetaData rsmd, int arrayColumn, JdbcToArrowConfig config) throws SQLException { Preconditions.checkNotNull(rsmd, "ResultSet MetaData object cannot be null"); Preconditions.checkNotNull(config, "Configuration must not be null"); Preconditions.checkArgument( - arrayColumn > 0, - "ResultSetMetaData columns start with 1; column cannot be less than 1"); + arrayColumn > 0, "ResultSetMetaData columns start with 1; column cannot be less than 1"); Preconditions.checkArgument( arrayColumn <= rsmd.getColumnCount(), "Column number cannot be more than the number of columns"); @@ -359,10 +368,10 @@ private static JdbcFieldInfo getJdbcFieldInfoForArraySubType( * Iterate the given JDBC {@link ResultSet} object to fetch the data and transpose it to populate * the given Arrow Vector objects. * - * @param rs ResultSet to use to fetch the data from underlying database - * @param root Arrow {@link VectorSchemaRoot} object to populate - * @param calendar The calendar to use when reading {@link Date}, {@link Time}, or {@link Timestamp} - * data types from the {@link ResultSet}, or null if not converting. + * @param rs ResultSet to use to fetch the data from underlying database + * @param root Arrow {@link VectorSchemaRoot} object to populate + * @param calendar The calendar to use when reading {@link Date}, {@link Time}, or {@link + * Timestamp} data types from the {@link ResultSet}, or null if not converting. * @throws SQLException on error */ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, Calendar calendar) @@ -373,29 +382,30 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, Calen jdbcToArrowVectors(rs, root, new JdbcToArrowConfig(new RootAllocator(0), calendar)); } - static boolean isColumnNullable(ResultSetMetaData resultSetMetadata, int index, JdbcFieldInfo info) - throws SQLException { + static boolean isColumnNullable( + ResultSetMetaData resultSetMetadata, int index, JdbcFieldInfo info) throws SQLException { int nullableValue; if (info != null && info.isNullable() != ResultSetMetaData.columnNullableUnknown) { nullableValue = info.isNullable(); } else { nullableValue = resultSetMetadata.isNullable(index); } - return nullableValue == ResultSetMetaData.columnNullable || - nullableValue == ResultSetMetaData.columnNullableUnknown; + return nullableValue == ResultSetMetaData.columnNullable + || nullableValue == ResultSetMetaData.columnNullableUnknown; } /** * Iterate the given JDBC {@link ResultSet} object to fetch the data and transpose it to populate * the given Arrow Vector objects. * - * @param rs ResultSet to use to fetch the data from underlying database - * @param root Arrow {@link VectorSchemaRoot} object to populate + * @param rs ResultSet to use to fetch the data from underlying database + * @param root Arrow {@link VectorSchemaRoot} object to populate * @param config The configuration to use when reading the data. * @throws SQLException on error * @throws JdbcConsumerException on error from VectorConsumer */ - public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, JdbcToArrowConfig config) + public static void jdbcToArrowVectors( + ResultSet rs, VectorSchemaRoot root, JdbcToArrowConfig config) throws SQLException, IOException { ResultSetMetaData rsmd = rs.getMetaData(); @@ -405,8 +415,13 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, JdbcT for (int i = 1; i <= columnCount; i++) { FieldVector vector = root.getVector(rsmd.getColumnLabel(i)); final JdbcFieldInfo columnFieldInfo = getJdbcFieldInfoForColumn(rsmd, i, config); - consumers[i - 1] = getConsumer( - vector.getField().getType(), i, isColumnNullable(rsmd, i, columnFieldInfo), vector, config); + consumers[i - 1] = + getConsumer( + vector.getField().getType(), + i, + isColumnNullable(rsmd, i, columnFieldInfo), + vector, + config); } CompositeJdbcConsumer compositeConsumer = null; @@ -439,18 +454,22 @@ public static void jdbcToArrowVectors(ResultSet rs, VectorSchemaRoot root, JdbcT } /** - * Default function used for JdbcConsumerFactory. This function gets a JdbcConsumer for the - * given column based on the Arrow type and provided vector. + * Default function used for JdbcConsumerFactory. This function gets a JdbcConsumer for the given + * column based on the Arrow type and provided vector. * - * @param arrowType Arrow type for the column. + * @param arrowType Arrow type for the column. * @param columnIndex Column index to fetch from the ResultSet - * @param nullable Whether the value is nullable or not - * @param vector Vector to store the consumed value - * @param config Associated JdbcToArrowConfig, used mainly for the Calendar. + * @param nullable Whether the value is nullable or not + * @param vector Vector to store the consumed value + * @param config Associated JdbcToArrowConfig, used mainly for the Calendar. * @return {@link JdbcConsumer} */ - public static JdbcConsumer getConsumer(ArrowType arrowType, int columnIndex, boolean nullable, - FieldVector vector, JdbcToArrowConfig config) { + public static JdbcConsumer getConsumer( + ArrowType arrowType, + int columnIndex, + boolean nullable, + FieldVector vector, + JdbcToArrowConfig config) { final Calendar calendar = config.getCalendar(); switch (arrowType.getTypeID()) { @@ -472,10 +491,11 @@ public static JdbcConsumer getConsumer(ArrowType arrowType, int columnIndex, boo case Decimal: final RoundingMode bigDecimalRoundingMode = config.getBigDecimalRoundingMode(); if (((ArrowType.Decimal) arrowType).getBitWidth() == 256) { - return Decimal256Consumer.createConsumer((Decimal256Vector) vector, columnIndex, nullable, - bigDecimalRoundingMode); + return Decimal256Consumer.createConsumer( + (Decimal256Vector) vector, columnIndex, nullable, bigDecimalRoundingMode); } else { - return DecimalConsumer.createConsumer((DecimalVector) vector, columnIndex, nullable, bigDecimalRoundingMode); + return DecimalConsumer.createConsumer( + (DecimalVector) vector, columnIndex, nullable, bigDecimalRoundingMode); } case FloatingPoint: switch (((ArrowType.FloatingPoint) arrowType).getPrecision()) { @@ -495,17 +515,25 @@ public static JdbcConsumer getConsumer(ArrowType arrowType, int columnIndex, boo case Date: return DateConsumer.createConsumer((DateDayVector) vector, columnIndex, nullable, calendar); case Time: - return TimeConsumer.createConsumer((TimeMilliVector) vector, columnIndex, nullable, calendar); + return TimeConsumer.createConsumer( + (TimeMilliVector) vector, columnIndex, nullable, calendar); case Timestamp: if (config.getCalendar() == null) { - return TimestampConsumer.createConsumer((TimeStampMilliVector) vector, columnIndex, nullable); + return TimestampConsumer.createConsumer( + (TimeStampMilliVector) vector, columnIndex, nullable); } else { - return TimestampTZConsumer.createConsumer((TimeStampMilliTZVector) vector, columnIndex, nullable, calendar); + return TimestampTZConsumer.createConsumer( + (TimeStampMilliTZVector) vector, columnIndex, nullable, calendar); } case List: FieldVector childVector = ((ListVector) vector).getDataVector(); - JdbcConsumer delegate = getConsumer(childVector.getField().getType(), JDBC_ARRAY_VALUE_COLUMN, - childVector.getField().isNullable(), childVector, config); + JdbcConsumer delegate = + getConsumer( + childVector.getField().getType(), + JDBC_ARRAY_VALUE_COLUMN, + childVector.getField().isNullable(), + childVector, + config); return ArrayConsumer.createConsumer((ListVector) vector, delegate, columnIndex, nullable); case Map: return MapConsumer.createConsumer((MapVector) vector, columnIndex, nullable); diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/BaseColumnBinder.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/BaseColumnBinder.java index f24f409072c0d..d7b62c43acf6f 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/BaseColumnBinder.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/BaseColumnBinder.java @@ -14,13 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.binder; import org.apache.arrow.vector.FieldVector; /** * Base class for ColumnBinder implementations. + * * @param The concrete FieldVector subtype. */ public abstract class BaseColumnBinder implements ColumnBinder { diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/BigIntBinder.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/BigIntBinder.java index fde4642ef90a5..b9dfcb0d6c956 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/BigIntBinder.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/BigIntBinder.java @@ -14,13 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.binder; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Types; - import org.apache.arrow.vector.BigIntVector; /** A column binder for 8-bit integers. */ @@ -34,7 +32,8 @@ public BigIntBinder(BigIntVector vector, int jdbcType) { } @Override - public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) throws SQLException { + public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) + throws SQLException { final long value = vector.getDataBuffer().getLong((long) rowIndex * BigIntVector.TYPE_WIDTH); statement.setLong(parameterIndex, value); } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/BitBinder.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/BitBinder.java index adae513e99e7c..c9db194f652ff 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/BitBinder.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/BitBinder.java @@ -14,13 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.binder; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Types; - import org.apache.arrow.vector.BitVector; /** A column binder for booleans. */ @@ -34,7 +32,8 @@ public BitBinder(BitVector vector, int jdbcType) { } @Override - public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) throws SQLException { + public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) + throws SQLException { // See BitVector#getBit final int byteIndex = rowIndex >> 3; final byte b = vector.getDataBuffer().getByte(byteIndex); diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/ColumnBinder.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/ColumnBinder.java index c2b1259e1424b..c38db68234ecf 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/ColumnBinder.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/ColumnBinder.java @@ -14,17 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.binder; import java.sql.PreparedStatement; import java.sql.SQLException; - import org.apache.arrow.vector.FieldVector; -/** - * A helper to bind values from a wrapped Arrow vector to a JDBC PreparedStatement. - */ +/** A helper to bind values from a wrapped Arrow vector to a JDBC PreparedStatement. */ public interface ColumnBinder { /** * Bind the given row to the given parameter. @@ -43,14 +39,10 @@ public interface ColumnBinder { */ int getJdbcType(); - /** - * Get the vector used by this binder. - */ + /** Get the vector used by this binder. */ FieldVector getVector(); - /** - * Create a column binder for a vector, using the default JDBC type code for null values. - */ + /** Create a column binder for a vector, using the default JDBC type code for null values. */ static ColumnBinder forVector(FieldVector vector) { return forVector(vector, /*jdbcType*/ null); } @@ -62,7 +54,8 @@ static ColumnBinder forVector(FieldVector vector) { * @param jdbcType The JDBC type code to use (or null to use the default). */ static ColumnBinder forVector(FieldVector vector, Integer jdbcType) { - final ColumnBinder binder = vector.getField().getType().accept(new ColumnBinderArrowTypeVisitor(vector, jdbcType)); + final ColumnBinder binder = + vector.getField().getType().accept(new ColumnBinderArrowTypeVisitor(vector, jdbcType)); if (vector.getField().isNullable()) { return new NullableColumnBinder(binder); } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/ColumnBinderArrowTypeVisitor.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/ColumnBinderArrowTypeVisitor.java index 7420a8c23dd48..30b2305f3f916 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/ColumnBinderArrowTypeVisitor.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/ColumnBinderArrowTypeVisitor.java @@ -14,14 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.binder; import java.sql.Types; import java.time.ZoneId; import java.util.Calendar; import java.util.TimeZone; - import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; import org.apache.arrow.vector.DateDayVector; @@ -50,8 +48,8 @@ /** * Visitor to create the base ColumnBinder for a vector. - *

    - * To handle null values, wrap the returned binder in a {@link NullableColumnBinder}. + * + *

    To handle null values, wrap the returned binder in a {@link NullableColumnBinder}. */ public class ColumnBinderArrowTypeVisitor implements ArrowType.ArrowTypeVisitor { private final FieldVector vector; @@ -111,17 +109,21 @@ public ColumnBinder visit(ArrowType.Int type) { } switch (type.getBitWidth()) { case 8: - return jdbcType == null ? new TinyIntBinder((TinyIntVector) vector) : - new TinyIntBinder((TinyIntVector) vector, jdbcType); + return jdbcType == null + ? new TinyIntBinder((TinyIntVector) vector) + : new TinyIntBinder((TinyIntVector) vector, jdbcType); case 16: - return jdbcType == null ? new SmallIntBinder((SmallIntVector) vector) : - new SmallIntBinder((SmallIntVector) vector, jdbcType); + return jdbcType == null + ? new SmallIntBinder((SmallIntVector) vector) + : new SmallIntBinder((SmallIntVector) vector, jdbcType); case 32: - return jdbcType == null ? new IntBinder((IntVector) vector) : - new IntBinder((IntVector) vector, jdbcType); + return jdbcType == null + ? new IntBinder((IntVector) vector) + : new IntBinder((IntVector) vector, jdbcType); case 64: - return jdbcType == null ? new BigIntBinder((BigIntVector) vector) : - new BigIntBinder((BigIntVector) vector, jdbcType); + return jdbcType == null + ? new BigIntBinder((BigIntVector) vector) + : new BigIntBinder((BigIntVector) vector, jdbcType); default: throw new UnsupportedOperationException("No column binder implemented for type " + type); } @@ -131,11 +133,13 @@ public ColumnBinder visit(ArrowType.Int type) { public ColumnBinder visit(ArrowType.FloatingPoint type) { switch (type.getPrecision()) { case SINGLE: - return jdbcType == null ? new Float4Binder((Float4Vector) vector) : - new Float4Binder((Float4Vector) vector, jdbcType); + return jdbcType == null + ? new Float4Binder((Float4Vector) vector) + : new Float4Binder((Float4Vector) vector, jdbcType); case DOUBLE: - return jdbcType == null ? new Float8Binder((Float8Vector) vector) : - new Float8Binder((Float8Vector) vector, jdbcType); + return jdbcType == null + ? new Float8Binder((Float8Vector) vector) + : new Float8Binder((Float8Vector) vector, jdbcType); default: throw new UnsupportedOperationException("No column binder implemented for type " + type); } @@ -144,61 +148,74 @@ public ColumnBinder visit(ArrowType.FloatingPoint type) { @Override public ColumnBinder visit(ArrowType.Utf8 type) { VarCharVector varChar = (VarCharVector) vector; - return jdbcType == null ? new VarCharBinder<>(varChar, Types.VARCHAR) : - new VarCharBinder<>(varChar, jdbcType); + return jdbcType == null + ? new VarCharBinder<>(varChar, Types.VARCHAR) + : new VarCharBinder<>(varChar, jdbcType); } @Override public ColumnBinder visit(ArrowType.Utf8View type) { - throw new UnsupportedOperationException("Column binder implemented for type " + type + " is not supported"); + throw new UnsupportedOperationException( + "Column binder implemented for type " + type + " is not supported"); } @Override public ColumnBinder visit(ArrowType.LargeUtf8 type) { LargeVarCharVector varChar = (LargeVarCharVector) vector; - return jdbcType == null ? new VarCharBinder<>(varChar, Types.LONGVARCHAR) : - new VarCharBinder<>(varChar, jdbcType); + return jdbcType == null + ? new VarCharBinder<>(varChar, Types.LONGVARCHAR) + : new VarCharBinder<>(varChar, jdbcType); } @Override public ColumnBinder visit(ArrowType.Binary type) { VarBinaryVector varBinary = (VarBinaryVector) vector; - return jdbcType == null ? new VarBinaryBinder<>(varBinary, Types.VARBINARY) : - new VarBinaryBinder<>(varBinary, jdbcType); + return jdbcType == null + ? new VarBinaryBinder<>(varBinary, Types.VARBINARY) + : new VarBinaryBinder<>(varBinary, jdbcType); } @Override public ColumnBinder visit(ArrowType.BinaryView type) { - throw new UnsupportedOperationException("Column binder implemented for type " + type + " is not supported"); + throw new UnsupportedOperationException( + "Column binder implemented for type " + type + " is not supported"); } @Override public ColumnBinder visit(ArrowType.LargeBinary type) { LargeVarBinaryVector varBinary = (LargeVarBinaryVector) vector; - return jdbcType == null ? new VarBinaryBinder<>(varBinary, Types.LONGVARBINARY) : - new VarBinaryBinder<>(varBinary, jdbcType); + return jdbcType == null + ? new VarBinaryBinder<>(varBinary, Types.LONGVARBINARY) + : new VarBinaryBinder<>(varBinary, jdbcType); } @Override public ColumnBinder visit(ArrowType.FixedSizeBinary type) { FixedSizeBinaryVector binary = (FixedSizeBinaryVector) vector; - return jdbcType == null ? new FixedSizeBinaryBinder(binary, Types.BINARY) : - new FixedSizeBinaryBinder(binary, jdbcType); + return jdbcType == null + ? new FixedSizeBinaryBinder(binary, Types.BINARY) + : new FixedSizeBinaryBinder(binary, jdbcType); } @Override public ColumnBinder visit(ArrowType.Bool type) { - return jdbcType == null ? new BitBinder((BitVector) vector) : new BitBinder((BitVector) vector, jdbcType); + return jdbcType == null + ? new BitBinder((BitVector) vector) + : new BitBinder((BitVector) vector, jdbcType); } @Override public ColumnBinder visit(ArrowType.Decimal type) { if (type.getBitWidth() == 128) { DecimalVector decimalVector = (DecimalVector) vector; - return jdbcType == null ? new Decimal128Binder(decimalVector) : new Decimal128Binder(decimalVector, jdbcType); + return jdbcType == null + ? new Decimal128Binder(decimalVector) + : new Decimal128Binder(decimalVector, jdbcType); } else if (type.getBitWidth() == 256) { Decimal256Vector decimalVector = (Decimal256Vector) vector; - return jdbcType == null ? new Decimal256Binder(decimalVector) : new Decimal256Binder(decimalVector, jdbcType); + return jdbcType == null + ? new Decimal256Binder(decimalVector) + : new Decimal256Binder(decimalVector, jdbcType); } throw new UnsupportedOperationException("No column binder implemented for type " + type); } @@ -207,11 +224,13 @@ public ColumnBinder visit(ArrowType.Decimal type) { public ColumnBinder visit(ArrowType.Date type) { switch (type.getUnit()) { case DAY: - return jdbcType == null ? new DateDayBinder((DateDayVector) vector) : - new DateDayBinder((DateDayVector) vector, /*calendar*/null, jdbcType); + return jdbcType == null + ? new DateDayBinder((DateDayVector) vector) + : new DateDayBinder((DateDayVector) vector, /*calendar*/ null, jdbcType); case MILLISECOND: - return jdbcType == null ? new DateMilliBinder((DateMilliVector) vector) : - new DateMilliBinder((DateMilliVector) vector, /*calendar*/null, jdbcType); + return jdbcType == null + ? new DateMilliBinder((DateMilliVector) vector) + : new DateMilliBinder((DateMilliVector) vector, /*calendar*/ null, jdbcType); default: throw new UnsupportedOperationException("No column binder implemented for type " + type); } @@ -221,17 +240,21 @@ public ColumnBinder visit(ArrowType.Date type) { public ColumnBinder visit(ArrowType.Time type) { switch (type.getUnit()) { case SECOND: - return jdbcType == null ? new Time32Binder((TimeSecVector) vector) : - new Time32Binder((TimeSecVector) vector, jdbcType); + return jdbcType == null + ? new Time32Binder((TimeSecVector) vector) + : new Time32Binder((TimeSecVector) vector, jdbcType); case MILLISECOND: - return jdbcType == null ? new Time32Binder((TimeMilliVector) vector) : - new Time32Binder((TimeMilliVector) vector, jdbcType); + return jdbcType == null + ? new Time32Binder((TimeMilliVector) vector) + : new Time32Binder((TimeMilliVector) vector, jdbcType); case MICROSECOND: - return jdbcType == null ? new Time64Binder((TimeMicroVector) vector) : - new Time64Binder((TimeMicroVector) vector, jdbcType); + return jdbcType == null + ? new Time64Binder((TimeMicroVector) vector) + : new Time64Binder((TimeMicroVector) vector, jdbcType); case NANOSECOND: - return jdbcType == null ? new Time64Binder((TimeNanoVector) vector) : - new Time64Binder((TimeNanoVector) vector, jdbcType); + return jdbcType == null + ? new Time64Binder((TimeNanoVector) vector) + : new Time64Binder((TimeNanoVector) vector, jdbcType); default: throw new UnsupportedOperationException("No column binder implemented for type " + type); } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/DateDayBinder.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/DateDayBinder.java index bc16790c8f391..b9eae464c8aa2 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/DateDayBinder.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/DateDayBinder.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.binder; import java.sql.Date; @@ -22,12 +21,9 @@ import java.sql.SQLException; import java.sql.Types; import java.util.Calendar; - import org.apache.arrow.vector.DateDayVector; -/** - * A column binder for 32-bit dates. - */ +/** A column binder for 32-bit dates. */ public class DateDayBinder extends BaseColumnBinder { private static final long MILLIS_PER_DAY = 86_400_000; private final Calendar calendar; @@ -46,7 +42,8 @@ public DateDayBinder(DateDayVector vector, Calendar calendar, int jdbcType) { } @Override - public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) throws SQLException { + public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) + throws SQLException { // TODO: multiply with overflow final long index = (long) rowIndex * DateDayVector.TYPE_WIDTH; final Date value = new Date(vector.getDataBuffer().getInt(index) * MILLIS_PER_DAY); diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/DateMilliBinder.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/DateMilliBinder.java index 5cb91b46ac179..f320391fbed5b 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/DateMilliBinder.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/DateMilliBinder.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.binder; import java.sql.Date; @@ -22,12 +21,9 @@ import java.sql.SQLException; import java.sql.Types; import java.util.Calendar; - import org.apache.arrow.vector.DateMilliVector; -/** - * A column binder for 64-bit dates. - */ +/** A column binder for 64-bit dates. */ public class DateMilliBinder extends BaseColumnBinder { private final Calendar calendar; @@ -39,14 +35,14 @@ public DateMilliBinder(DateMilliVector vector, Calendar calendar) { this(vector, calendar, Types.DATE); } - public DateMilliBinder(DateMilliVector vector, Calendar calendar, int jdbcType) { super(vector, jdbcType); this.calendar = calendar; } @Override - public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) throws SQLException { + public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) + throws SQLException { final long index = (long) rowIndex * DateMilliVector.TYPE_WIDTH; final Date value = new Date(vector.getDataBuffer().getLong(index)); if (calendar == null) { diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/Decimal128Binder.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/Decimal128Binder.java index 9e9d0e4fdb25b..07ef52f2e594c 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/Decimal128Binder.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/Decimal128Binder.java @@ -14,20 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.binder; import java.math.BigDecimal; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Types; - import org.apache.arrow.vector.DecimalVector; import org.apache.arrow.vector.util.DecimalUtility; -/** - * A binder for 128-bit decimals. - */ +/** A binder for 128-bit decimals. */ public class Decimal128Binder extends BaseColumnBinder { public Decimal128Binder(DecimalVector vector) { this(vector, Types.DECIMAL); @@ -38,9 +34,11 @@ public Decimal128Binder(DecimalVector vector, int jdbcType) { } @Override - public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) throws SQLException { - final BigDecimal value = DecimalUtility.getBigDecimalFromArrowBuf( - vector.getDataBuffer(), rowIndex, vector.getScale(), DecimalVector.TYPE_WIDTH); + public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) + throws SQLException { + final BigDecimal value = + DecimalUtility.getBigDecimalFromArrowBuf( + vector.getDataBuffer(), rowIndex, vector.getScale(), DecimalVector.TYPE_WIDTH); statement.setBigDecimal(parameterIndex, value); } } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/Decimal256Binder.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/Decimal256Binder.java index bd29e083b4513..5a4222f6b84db 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/Decimal256Binder.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/Decimal256Binder.java @@ -14,20 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.binder; import java.math.BigDecimal; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Types; - import org.apache.arrow.vector.Decimal256Vector; import org.apache.arrow.vector.util.DecimalUtility; -/** - * A binder for 256-bit decimals. - */ +/** A binder for 256-bit decimals. */ public class Decimal256Binder extends BaseColumnBinder { public Decimal256Binder(Decimal256Vector vector) { this(vector, Types.DECIMAL); @@ -38,9 +34,11 @@ public Decimal256Binder(Decimal256Vector vector, int jdbcType) { } @Override - public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) throws SQLException { - final BigDecimal value = DecimalUtility.getBigDecimalFromArrowBuf( - vector.getDataBuffer(), rowIndex, vector.getScale(), Decimal256Vector.TYPE_WIDTH); + public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) + throws SQLException { + final BigDecimal value = + DecimalUtility.getBigDecimalFromArrowBuf( + vector.getDataBuffer(), rowIndex, vector.getScale(), Decimal256Vector.TYPE_WIDTH); statement.setBigDecimal(parameterIndex, value); } } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/FixedSizeBinaryBinder.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/FixedSizeBinaryBinder.java index 7edc5e4532985..4f74b1fa8cfd4 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/FixedSizeBinaryBinder.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/FixedSizeBinaryBinder.java @@ -14,22 +14,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.binder; import java.sql.PreparedStatement; import java.sql.SQLException; - import org.apache.arrow.vector.FixedSizeBinaryVector; -/** - * A binder for fixed-width binary types. - */ +/** A binder for fixed-width binary types. */ public class FixedSizeBinaryBinder extends BaseColumnBinder { /** * Create a binder for the given vector using the given JDBC type for null values. * - * @param vector The vector to draw values from. + * @param vector The vector to draw values from. * @param jdbcType The JDBC type code. */ public FixedSizeBinaryBinder(FixedSizeBinaryVector vector, int jdbcType) { @@ -37,9 +33,12 @@ public FixedSizeBinaryBinder(FixedSizeBinaryVector vector, int jdbcType) { } @Override - public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) throws SQLException { + public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) + throws SQLException { byte[] binaryData = new byte[vector.getByteWidth()]; - vector.getDataBuffer().getBytes((long) rowIndex * binaryData.length, binaryData, 0, binaryData.length); + vector + .getDataBuffer() + .getBytes((long) rowIndex * binaryData.length, binaryData, 0, binaryData.length); statement.setBytes(parameterIndex, binaryData); } } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/Float4Binder.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/Float4Binder.java index a471c1ebadd66..466a67a2dbc89 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/Float4Binder.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/Float4Binder.java @@ -14,18 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.binder; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Types; - import org.apache.arrow.vector.Float4Vector; -/** - * A binder for 32-bit floats. - */ +/** A binder for 32-bit floats. */ public class Float4Binder extends BaseColumnBinder { public Float4Binder(Float4Vector vector) { this(vector, Types.REAL); @@ -36,7 +32,8 @@ public Float4Binder(Float4Vector vector, int jdbcType) { } @Override - public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) throws SQLException { + public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) + throws SQLException { final float value = vector.getDataBuffer().getFloat((long) rowIndex * Float4Vector.TYPE_WIDTH); statement.setFloat(parameterIndex, value); } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/Float8Binder.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/Float8Binder.java index 4710c3b59860d..222bebf115372 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/Float8Binder.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/Float8Binder.java @@ -14,18 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.binder; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Types; - import org.apache.arrow.vector.Float8Vector; -/** - * A binder for 64-bit floats. - */ +/** A binder for 64-bit floats. */ public class Float8Binder extends BaseColumnBinder { public Float8Binder(Float8Vector vector) { this(vector, Types.DOUBLE); @@ -36,8 +32,10 @@ public Float8Binder(Float8Vector vector, int jdbcType) { } @Override - public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) throws SQLException { - final double value = vector.getDataBuffer().getDouble((long) rowIndex * Float8Vector.TYPE_WIDTH); + public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) + throws SQLException { + final double value = + vector.getDataBuffer().getDouble((long) rowIndex * Float8Vector.TYPE_WIDTH); statement.setDouble(parameterIndex, value); } } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/IntBinder.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/IntBinder.java index 7d47f585a39d9..6b49eeb5352b1 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/IntBinder.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/IntBinder.java @@ -14,13 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.binder; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Types; - import org.apache.arrow.vector.IntVector; /** A column binder for 32-bit integers. */ @@ -34,7 +32,8 @@ public IntBinder(IntVector vector, int jdbcType) { } @Override - public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) throws SQLException { + public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) + throws SQLException { final int value = vector.getDataBuffer().getInt((long) rowIndex * IntVector.TYPE_WIDTH); statement.setInt(parameterIndex, value); } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/ListBinder.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/ListBinder.java index b8aa61234f4e9..25172c0c1f0aa 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/ListBinder.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/ListBinder.java @@ -14,21 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.binder; import java.lang.reflect.Array; import java.util.ArrayList; import java.util.Arrays; - import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.complex.ListVector; import org.apache.arrow.vector.complex.impl.UnionListReader; import org.apache.arrow.vector.util.Text; -/** - * A column binder for list of primitive values. - */ +/** A column binder for list of primitive values. */ public class ListBinder extends BaseColumnBinder { private final UnionListReader listReader; @@ -52,7 +48,9 @@ public ListBinder(ListVector vector, int jdbcType) { try { arrayElementClass = dataVectorClass.getMethod("getObject", Integer.TYPE).getReturnType(); } catch (NoSuchMethodException e) { - final String message = String.format("Issue to determine type for getObject method of data vector class %s ", + final String message = + String.format( + "Issue to determine type for getObject method of data vector class %s ", dataVectorClass.getName()); throw new RuntimeException(message); } @@ -60,7 +58,8 @@ public ListBinder(ListVector vector, int jdbcType) { } @Override - public void bind(java.sql.PreparedStatement statement, int parameterIndex, int rowIndex)throws java.sql.SQLException { + public void bind(java.sql.PreparedStatement statement, int parameterIndex, int rowIndex) + throws java.sql.SQLException { listReader.setPosition(rowIndex); ArrayList sourceArray = (ArrayList) listReader.readObject(); Object array; @@ -69,7 +68,9 @@ public void bind(java.sql.PreparedStatement statement, int parameterIndex, int r Arrays.setAll((Object[]) array, sourceArray::get); } else { array = new String[sourceArray.size()]; - Arrays.setAll((Object[]) array, idx -> sourceArray.get(idx) != null ? sourceArray.get(idx).toString() : null); + Arrays.setAll( + (Object[]) array, + idx -> sourceArray.get(idx) != null ? sourceArray.get(idx).toString() : null); } statement.setObject(parameterIndex, array); } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/MapBinder.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/MapBinder.java index 07391eb7cbfb4..e94f186453581 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/MapBinder.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/MapBinder.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.binder; import java.sql.PreparedStatement; @@ -23,16 +22,13 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Objects; - import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.impl.UnionMapReader; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.util.JsonStringHashMap; -/** - * A column binder for map of primitive values. - */ +/** A column binder for map of primitive values. */ public class MapBinder extends BaseColumnBinder { private UnionMapReader reader; @@ -58,8 +54,8 @@ public MapBinder(MapVector vector, int jdbcType) { } List keyValueFields = Objects.requireNonNull(structField.get(0)).getChildren(); if (keyValueFields.size() != 2) { - throw new IllegalArgumentException("Expected two children fields " + - "inside nested Struct field in Map"); + throw new IllegalArgumentException( + "Expected two children fields " + "inside nested Struct field in Map"); } ArrowType keyType = Objects.requireNonNull(keyValueFields.get(0)).getType(); ArrowType valueType = Objects.requireNonNull(keyValueFields.get(1)).getType(); @@ -68,15 +64,16 @@ public MapBinder(MapVector vector, int jdbcType) { } @Override - public void bind(PreparedStatement statement, - int parameterIndex, int rowIndex) throws SQLException { + public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) + throws SQLException { reader.setPosition(rowIndex); LinkedHashMap tags = new JsonStringHashMap<>(); while (reader.next()) { Object key = reader.key().readObject(); Object value = reader.value().readObject(); - tags.put(isTextKey && key != null ? key.toString() : key, - isTextValue && value != null ? value.toString() : value); + tags.put( + isTextKey && key != null ? key.toString() : key, + isTextValue && value != null ? value.toString() : value); } switch (jdbcType) { case Types.VARCHAR: diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/NullableColumnBinder.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/NullableColumnBinder.java index 123b587ca50d4..bf5288b173341 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/NullableColumnBinder.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/NullableColumnBinder.java @@ -14,17 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.binder; import java.sql.PreparedStatement; import java.sql.SQLException; - import org.apache.arrow.vector.FieldVector; -/** - * A ColumnBinder that checks for nullability before deferring to a type-specific binder. - */ +/** A ColumnBinder that checks for nullability before deferring to a type-specific binder. */ public class NullableColumnBinder implements ColumnBinder { private final ColumnBinder wrapped; @@ -33,7 +29,8 @@ public NullableColumnBinder(ColumnBinder wrapped) { } @Override - public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) throws SQLException { + public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) + throws SQLException { if (wrapped.getVector().isNull(rowIndex)) { statement.setNull(parameterIndex, wrapped.getJdbcType()); } else { diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/SmallIntBinder.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/SmallIntBinder.java index f9d744b9f5497..aa636c9336f55 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/SmallIntBinder.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/SmallIntBinder.java @@ -14,13 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.binder; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Types; - import org.apache.arrow.vector.SmallIntVector; /** A column binder for 8-bit integers. */ @@ -34,8 +32,10 @@ public SmallIntBinder(SmallIntVector vector, int jdbcType) { } @Override - public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) throws SQLException { - final short value = vector.getDataBuffer().getShort((short) rowIndex * SmallIntVector.TYPE_WIDTH); + public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) + throws SQLException { + final short value = + vector.getDataBuffer().getShort((short) rowIndex * SmallIntVector.TYPE_WIDTH); statement.setShort(parameterIndex, value); } } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/Time32Binder.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/Time32Binder.java index 5dc7e3f513f97..4e09c3be23264 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/Time32Binder.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/Time32Binder.java @@ -14,21 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.binder; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Time; import java.sql.Types; - import org.apache.arrow.vector.BaseFixedWidthVector; import org.apache.arrow.vector.TimeMilliVector; import org.apache.arrow.vector.TimeSecVector; -/** - * A binder for 32-bit time types. - */ +/** A binder for 32-bit time types. */ public class Time32Binder extends BaseColumnBinder { private static final long TYPE_WIDTH = 4; @@ -43,11 +39,11 @@ public Time32Binder(TimeMilliVector vector) { } public Time32Binder(TimeSecVector vector, int jdbcType) { - this(vector, /*factor*/1_000, jdbcType); + this(vector, /*factor*/ 1_000, jdbcType); } public Time32Binder(TimeMilliVector vector, int jdbcType) { - this(vector, /*factor*/1, jdbcType); + this(vector, /*factor*/ 1, jdbcType); } Time32Binder(BaseFixedWidthVector vector, long factor, int jdbcType) { @@ -56,7 +52,8 @@ public Time32Binder(TimeMilliVector vector, int jdbcType) { } @Override - public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) throws SQLException { + public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) + throws SQLException { // TODO: multiply with overflow // TODO: take in a Calendar as well? final Time value = new Time(vector.getDataBuffer().getInt(rowIndex * TYPE_WIDTH) * factor); diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/Time64Binder.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/Time64Binder.java index 8d62ae0eb36df..01c85fb32f1b5 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/Time64Binder.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/Time64Binder.java @@ -14,21 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.binder; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Time; import java.sql.Types; - import org.apache.arrow.vector.BaseFixedWidthVector; import org.apache.arrow.vector.TimeMicroVector; import org.apache.arrow.vector.TimeNanoVector; -/** - * A binder for 64-bit time types. - */ +/** A binder for 64-bit time types. */ public class Time64Binder extends BaseColumnBinder { private static final long TYPE_WIDTH = 8; @@ -43,11 +39,11 @@ public Time64Binder(TimeNanoVector vector) { } public Time64Binder(TimeMicroVector vector, int jdbcType) { - this(vector, /*factor*/1_000, jdbcType); + this(vector, /*factor*/ 1_000, jdbcType); } public Time64Binder(TimeNanoVector vector, int jdbcType) { - this(vector, /*factor*/1_000_000, jdbcType); + this(vector, /*factor*/ 1_000_000, jdbcType); } Time64Binder(BaseFixedWidthVector vector, long factor, int jdbcType) { @@ -56,7 +52,8 @@ public Time64Binder(TimeNanoVector vector, int jdbcType) { } @Override - public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) throws SQLException { + public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) + throws SQLException { // TODO: option to throw on truncation (vendor Guava IntMath#multiply) final Time value = new Time(vector.getDataBuffer().getLong(rowIndex * TYPE_WIDTH) / factor); statement.setTime(parameterIndex, value); diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/TimeStampBinder.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/TimeStampBinder.java index 6677e5909901a..942d7ae58dcd5 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/TimeStampBinder.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/TimeStampBinder.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.binder; import java.sql.PreparedStatement; @@ -22,7 +21,6 @@ import java.sql.Timestamp; import java.sql.Types; import java.util.Calendar; - import org.apache.arrow.vector.TimeStampVector; import org.apache.arrow.vector.types.pojo.ArrowType; @@ -32,15 +30,17 @@ public class TimeStampBinder extends BaseColumnBinder { private final long unitsPerSecond; private final long nanosPerUnit; - /** - * Create a binder for a timestamp vector using the default JDBC type code. - */ + /** Create a binder for a timestamp vector using the default JDBC type code. */ public TimeStampBinder(TimeStampVector vector, Calendar calendar) { - this(vector, calendar, isZoned(vector.getField().getType()) ? Types.TIMESTAMP_WITH_TIMEZONE : Types.TIMESTAMP); + this( + vector, + calendar, + isZoned(vector.getField().getType()) ? Types.TIMESTAMP_WITH_TIMEZONE : Types.TIMESTAMP); } /** * Create a binder for a timestamp vector. + * * @param vector The vector to pull values from. * @param calendar Optionally, the calendar to pass to JDBC. * @param jdbcType The JDBC type code to use for null values. @@ -73,19 +73,23 @@ public TimeStampBinder(TimeStampVector vector, Calendar calendar, int jdbcType) } @Override - public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) throws SQLException { + public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) + throws SQLException { // TODO: option to throw on truncation (vendor Guava IntMath#multiply) or overflow - final long rawValue = vector.getDataBuffer().getLong((long) rowIndex * TimeStampVector.TYPE_WIDTH); + final long rawValue = + vector.getDataBuffer().getLong((long) rowIndex * TimeStampVector.TYPE_WIDTH); final long seconds = rawValue / unitsPerSecond; final int nanos = (int) ((rawValue - (seconds * unitsPerSecond)) * nanosPerUnit); final Timestamp value = new Timestamp(seconds * 1_000); value.setNanos(nanos); if (calendar != null) { - // Timestamp == Date == UTC timestamp (confusingly). Arrow's timestamp with timezone is a UTC value with a + // Timestamp == Date == UTC timestamp (confusingly). Arrow's timestamp with timezone is a UTC + // value with a // zone offset, so we don't need to do any conversion. statement.setTimestamp(parameterIndex, value, calendar); } else { - // Arrow timestamp without timezone isn't strictly convertible to any timezone. So this is technically wrong, + // Arrow timestamp without timezone isn't strictly convertible to any timezone. So this is + // technically wrong, // but there is no 'correct' interpretation here. The application should provide a calendar. statement.setTimestamp(parameterIndex, value); } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/TinyIntBinder.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/TinyIntBinder.java index f51d139be863a..0580456d37983 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/TinyIntBinder.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/TinyIntBinder.java @@ -14,13 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.binder; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Types; - import org.apache.arrow.vector.TinyIntVector; /** A column binder for 8-bit integers. */ @@ -34,7 +32,8 @@ public TinyIntBinder(TinyIntVector vector, int jdbcType) { } @Override - public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) throws SQLException { + public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) + throws SQLException { final byte value = vector.getDataBuffer().getByte((long) rowIndex * TinyIntVector.TYPE_WIDTH); statement.setByte(parameterIndex, value); } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/VarBinaryBinder.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/VarBinaryBinder.java index a94cff6a00496..41807efc611b1 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/VarBinaryBinder.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/VarBinaryBinder.java @@ -14,12 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.binder; import java.sql.PreparedStatement; import java.sql.SQLException; - import org.apache.arrow.memory.util.ArrowBufPointer; import org.apache.arrow.vector.ElementAddressableVector; import org.apache.arrow.vector.FieldVector; @@ -29,13 +27,14 @@ * * @param The binary vector. */ -public class VarBinaryBinder extends BaseColumnBinder { +public class VarBinaryBinder + extends BaseColumnBinder { private final ArrowBufPointer element; /** * Create a binder for the given vector using the given JDBC type for null values. * - * @param vector The vector to draw values from. + * @param vector The vector to draw values from. * @param jdbcType The JDBC type code. */ public VarBinaryBinder(T vector, int jdbcType) { @@ -44,15 +43,18 @@ public VarBinaryBinder(T vector, int jdbcType) { } @Override - public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) throws SQLException { + public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) + throws SQLException { vector.getDataPointer(rowIndex, element); if (element.getBuf() == null) { statement.setNull(parameterIndex, jdbcType); return; } if (element.getLength() > (long) Integer.MAX_VALUE) { - final String message = String.format("Length of value at index %d (%d) exceeds Integer.MAX_VALUE", - rowIndex, element.getLength()); + final String message = + String.format( + "Length of value at index %d (%d) exceeds Integer.MAX_VALUE", + rowIndex, element.getLength()); throw new RuntimeException(message); } byte[] binaryData = new byte[(int) element.getLength()]; diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/VarCharBinder.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/VarCharBinder.java index 73bd55981490b..926e1da28c9a0 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/VarCharBinder.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/VarCharBinder.java @@ -14,13 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.binder; import java.nio.charset.StandardCharsets; import java.sql.PreparedStatement; import java.sql.SQLException; - import org.apache.arrow.memory.util.ArrowBufPointer; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.VariableWidthVector; @@ -30,13 +28,14 @@ * * @param The text vector. */ -public class VarCharBinder extends BaseColumnBinder { +public class VarCharBinder + extends BaseColumnBinder { private final ArrowBufPointer element; /** * Create a binder for the given vector using the given JDBC type for null values. * - * @param vector The vector to draw values from. + * @param vector The vector to draw values from. * @param jdbcType The JDBC type code. */ public VarCharBinder(T vector, int jdbcType) { @@ -45,15 +44,18 @@ public VarCharBinder(T vector, int jdbcType) { } @Override - public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) throws SQLException { + public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) + throws SQLException { vector.getDataPointer(rowIndex, element); if (element.getBuf() == null) { statement.setNull(parameterIndex, jdbcType); return; } if (element.getLength() > (long) Integer.MAX_VALUE) { - final String message = String.format("Length of value at index %d (%d) exceeds Integer.MAX_VALUE", - rowIndex, element.getLength()); + final String message = + String.format( + "Length of value at index %d (%d) exceeds Integer.MAX_VALUE", + rowIndex, element.getLength()); throw new RuntimeException(message); } byte[] utf8Bytes = new byte[(int) element.getLength()]; diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/package-info.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/package-info.java index 4f8936e0c27bf..945c3c9f84fa8 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/package-info.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/binder/package-info.java @@ -15,8 +15,5 @@ * limitations under the License. */ -/** - * Utilities to bind Arrow data as JDBC prepared statement parameters. - */ - +/** Utilities to bind Arrow data as JDBC prepared statement parameters. */ package org.apache.arrow.adapter.jdbc.binder; diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/ArrayConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/ArrayConsumer.java index 2f18b8a416d34..4676e8204eed4 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/ArrayConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/ArrayConsumer.java @@ -14,29 +14,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.consumer; import java.io.IOException; import java.sql.Array; import java.sql.ResultSet; import java.sql.SQLException; - import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.complex.ListVector; /** - * Consumer which consume array type values from {@link ResultSet}. - * Write the data to {@link org.apache.arrow.vector.complex.ListVector}. + * Consumer which consume array type values from {@link ResultSet}. Write the data to {@link + * org.apache.arrow.vector.complex.ListVector}. */ public abstract class ArrayConsumer extends BaseConsumer { - /** - * Creates a consumer for {@link ListVector}. - */ + /** Creates a consumer for {@link ListVector}. */ public static ArrayConsumer createConsumer( - ListVector vector, JdbcConsumer delegate, int index, boolean nullable) { + ListVector vector, JdbcConsumer delegate, int index, boolean nullable) { if (nullable) { return new ArrayConsumer.NullableArrayConsumer(vector, delegate, index); } else { @@ -50,9 +46,7 @@ public static ArrayConsumer createConsumer( protected int innerVectorIndex = 0; - /** - * Instantiate a ArrayConsumer. - */ + /** Instantiate a ArrayConsumer. */ public ArrayConsumer(ListVector vector, JdbcConsumer delegate, int index) { super(vector, index); this.delegate = delegate; @@ -81,14 +75,10 @@ void ensureInnerVectorCapacity(int targetCapacity) { } } - /** - * Nullable consumer for {@link ListVector}. - */ + /** Nullable consumer for {@link ListVector}. */ static class NullableArrayConsumer extends ArrayConsumer { - /** - * Instantiate a nullable array consumer. - */ + /** Instantiate a nullable array consumer. */ public NullableArrayConsumer(ListVector vector, JdbcConsumer delegate, int index) { super(vector, delegate, index); } @@ -113,14 +103,10 @@ public void consume(ResultSet resultSet) throws SQLException, IOException { } } - /** - * Non-nullable consumer for {@link ListVector}. - */ + /** Non-nullable consumer for {@link ListVector}. */ static class NonNullableArrayConsumer extends ArrayConsumer { - /** - * Instantiate a nullable array consumer. - */ + /** Instantiate a nullable array consumer. */ public NonNullableArrayConsumer(ListVector vector, JdbcConsumer delegate, int index) { super(vector, delegate, index); } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BaseConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BaseConsumer.java index 2db128d3e2b2d..9ca3c98a7eb98 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BaseConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BaseConsumer.java @@ -14,13 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.consumer; import org.apache.arrow.vector.ValueVector; /** * Base class for all consumers. + * * @param vector type. */ public abstract class BaseConsumer implements JdbcConsumer { @@ -33,6 +33,7 @@ public abstract class BaseConsumer implements JdbcConsume /** * Constructs a new consumer. + * * @param vector the underlying vector for the consumer. * @param index the column id for the consumer. */ diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BigIntConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BigIntConsumer.java index 19c8efa91719f..b7c547a9391b6 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BigIntConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BigIntConsumer.java @@ -14,24 +14,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.consumer; import java.sql.ResultSet; import java.sql.SQLException; - import org.apache.arrow.vector.BigIntVector; /** - * Consumer which consume bigint type values from {@link ResultSet}. - * Write the data to {@link org.apache.arrow.vector.BigIntVector}. + * Consumer which consume bigint type values from {@link ResultSet}. Write the data to {@link + * org.apache.arrow.vector.BigIntVector}. */ public class BigIntConsumer { - /** - * Creates a consumer for {@link BigIntVector}. - */ - public static JdbcConsumer createConsumer(BigIntVector vector, int index, boolean nullable) { + /** Creates a consumer for {@link BigIntVector}. */ + public static JdbcConsumer createConsumer( + BigIntVector vector, int index, boolean nullable) { if (nullable) { return new NullableBigIntConsumer(vector, index); } else { @@ -39,14 +36,10 @@ public static JdbcConsumer createConsumer(BigIntVector vector, int } } - /** - * Nullable consumer for big int. - */ + /** Nullable consumer for big int. */ static class NullableBigIntConsumer extends BaseConsumer { - /** - * Instantiate a BigIntConsumer. - */ + /** Instantiate a BigIntConsumer. */ public NullableBigIntConsumer(BigIntVector vector, int index) { super(vector, index); } @@ -63,14 +56,10 @@ public void consume(ResultSet resultSet) throws SQLException { } } - /** - * Non-nullable consumer for big int. - */ + /** Non-nullable consumer for big int. */ static class NonNullableBigIntConsumer extends BaseConsumer { - /** - * Instantiate a BigIntConsumer. - */ + /** Instantiate a BigIntConsumer. */ public NonNullableBigIntConsumer(BigIntVector vector, int index) { super(vector, index); } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumer.java index 538d161f9e9c7..edbc6360df6bf 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumer.java @@ -14,27 +14,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.consumer; import java.io.IOException; import java.io.InputStream; import java.sql.ResultSet; import java.sql.SQLException; - import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.vector.BitVectorHelper; import org.apache.arrow.vector.VarBinaryVector; /** - * Consumer which consume binary type values from {@link ResultSet}. - * Write the data to {@link org.apache.arrow.vector.VarBinaryVector}. + * Consumer which consume binary type values from {@link ResultSet}. Write the data to {@link + * org.apache.arrow.vector.VarBinaryVector}. */ public abstract class BinaryConsumer extends BaseConsumer { - /** - * Creates a consumer for {@link VarBinaryVector}. - */ + /** Creates a consumer for {@link VarBinaryVector}. */ public static BinaryConsumer createConsumer(VarBinaryVector vector, int index, boolean nullable) { if (nullable) { return new NullableBinaryConsumer(vector, index); @@ -45,9 +41,7 @@ public static BinaryConsumer createConsumer(VarBinaryVector vector, int index, b private final byte[] reuseBytes = new byte[1024]; - /** - * Instantiate a BinaryConsumer. - */ + /** Instantiate a BinaryConsumer. */ public BinaryConsumer(VarBinaryVector vector, int index) { super(vector, index); if (vector != null) { @@ -55,9 +49,7 @@ public BinaryConsumer(VarBinaryVector vector, int index) { } } - /** - * consume a InputStream. - */ + /** consume a InputStream. */ public void consume(InputStream is) throws IOException { if (is != null) { while (currentIndex >= vector.getValueCapacity()) { @@ -74,7 +66,8 @@ public void consume(InputStream is) throws IOException { vector.getDataBuffer().setBytes(startOffset + dataLength, reuseBytes, 0, read); dataLength += read; } - offsetBuffer.setInt((currentIndex + 1) * ((long) VarBinaryVector.OFFSET_WIDTH), startOffset + dataLength); + offsetBuffer.setInt( + (currentIndex + 1) * ((long) VarBinaryVector.OFFSET_WIDTH), startOffset + dataLength); BitVectorHelper.setBit(vector.getValidityBuffer(), currentIndex); vector.setLastSet(currentIndex); } @@ -91,14 +84,10 @@ public void resetValueVector(VarBinaryVector vector) { this.currentIndex = 0; } - /** - * Consumer for nullable binary data. - */ + /** Consumer for nullable binary data. */ static class NullableBinaryConsumer extends BinaryConsumer { - - /** - * Instantiate a BinaryConsumer. - */ + + /** Instantiate a BinaryConsumer. */ public NullableBinaryConsumer(VarBinaryVector vector, int index) { super(vector, index); } @@ -113,14 +102,10 @@ public void consume(ResultSet resultSet) throws SQLException, IOException { } } - /** - * Consumer for non-nullable binary data. - */ + /** Consumer for non-nullable binary data. */ static class NonNullableBinaryConsumer extends BinaryConsumer { - /** - * Instantiate a BinaryConsumer. - */ + /** Instantiate a BinaryConsumer. */ public NonNullableBinaryConsumer(VarBinaryVector vector, int index) { super(vector, index); } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BitConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BitConsumer.java index d2d94d0a40e2f..287b9509b5054 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BitConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BitConsumer.java @@ -14,24 +14,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.consumer; import java.sql.ResultSet; import java.sql.SQLException; - import org.apache.arrow.vector.BitVector; /** - * Consumer which consume bit type values from {@link ResultSet}. - * Write the data to {@link BitVector}. + * Consumer which consume bit type values from {@link ResultSet}. Write the data to {@link + * BitVector}. */ public class BitConsumer { - /** - * Creates a consumer for {@link BitVector}. - */ - public static JdbcConsumer createConsumer(BitVector vector, int index, boolean nullable) { + /** Creates a consumer for {@link BitVector}. */ + public static JdbcConsumer createConsumer( + BitVector vector, int index, boolean nullable) { if (nullable) { return new NullableBitConsumer(vector, index); } else { @@ -39,14 +36,10 @@ public static JdbcConsumer createConsumer(BitVector vector, int index } } - /** - * Nullable consumer for {@link BitVector}. - */ + /** Nullable consumer for {@link BitVector}. */ static class NullableBitConsumer extends BaseConsumer { - /** - * Instantiate a BitConsumer. - */ + /** Instantiate a BitConsumer. */ public NullableBitConsumer(BitVector vector, int index) { super(vector, index); } @@ -63,14 +56,10 @@ public void consume(ResultSet resultSet) throws SQLException { } } - /** - * Non-nullable consumer for {@link BitVector}. - */ + /** Non-nullable consumer for {@link BitVector}. */ static class NonNullableBitConsumer extends BaseConsumer { - /** - * Instantiate a BitConsumer. - */ + /** Instantiate a BitConsumer. */ public NonNullableBitConsumer(BitVector vector, int index) { super(vector, index); } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BlobConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BlobConsumer.java index e57ecdf91707a..a4fc789494e0f 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BlobConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/BlobConsumer.java @@ -14,19 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.consumer; import java.io.IOException; import java.sql.Blob; import java.sql.ResultSet; import java.sql.SQLException; - import org.apache.arrow.vector.VarBinaryVector; /** - * Consumer which consume blob type values from {@link ResultSet}. - * Write the data to {@link VarBinaryVector}. + * Consumer which consume blob type values from {@link ResultSet}. Write the data to {@link + * VarBinaryVector}. */ public class BlobConsumer extends BaseConsumer { @@ -34,17 +32,12 @@ public class BlobConsumer extends BaseConsumer { private final boolean nullable; - /** - * Creates a consumer for {@link VarBinaryVector}. - */ - public static BlobConsumer createConsumer( - BinaryConsumer delegate, int index, boolean nullable) { + /** Creates a consumer for {@link VarBinaryVector}. */ + public static BlobConsumer createConsumer(BinaryConsumer delegate, int index, boolean nullable) { return new BlobConsumer(delegate, index, nullable); } - /** - * Instantiate a BlobConsumer. - */ + /** Instantiate a BlobConsumer. */ public BlobConsumer(BinaryConsumer delegate, int index, boolean nullable) { super(null, index); this.delegate = delegate; diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/ClobConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/ClobConsumer.java index 3ed0c2d3cbb2f..7deba1cbffebd 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/ClobConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/ClobConsumer.java @@ -14,28 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.consumer; import java.nio.charset.StandardCharsets; import java.sql.Clob; import java.sql.ResultSet; import java.sql.SQLException; - import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.util.MemoryUtil; import org.apache.arrow.vector.BitVectorHelper; import org.apache.arrow.vector.VarCharVector; /** - * Consumer which consume clob type values from {@link ResultSet}. - * Write the data to {@link org.apache.arrow.vector.VarCharVector}. + * Consumer which consume clob type values from {@link ResultSet}. Write the data to {@link + * org.apache.arrow.vector.VarCharVector}. */ public abstract class ClobConsumer extends BaseConsumer { - /** - * Creates a consumer for {@link VarCharVector}. - */ + /** Creates a consumer for {@link VarCharVector}. */ public static ClobConsumer createConsumer(VarCharVector vector, int index, boolean nullable) { if (nullable) { return new NullableClobConsumer(vector, index); @@ -46,9 +42,7 @@ public static ClobConsumer createConsumer(VarCharVector vector, int index, boole private static final int BUFFER_SIZE = 256; - /** - * Instantiate a ClobConsumer. - */ + /** Instantiate a ClobConsumer. */ public ClobConsumer(VarCharVector vector, int index) { super(vector, index); if (vector != null) { @@ -63,14 +57,10 @@ public void resetValueVector(VarCharVector vector) { this.currentIndex = 0; } - /** - * Nullable consumer for clob data. - */ + /** Nullable consumer for clob data. */ static class NullableClobConsumer extends ClobConsumer { - - /** - * Instantiate a ClobConsumer. - */ + + /** Instantiate a ClobConsumer. */ public NullableClobConsumer(VarCharVector vector, int index) { super(vector, index); } @@ -97,11 +87,11 @@ public void consume(ResultSet resultSet) throws SQLException { vector.reallocDataBuffer(); } MemoryUtil.UNSAFE.copyMemory( - bytes, - MemoryUtil.BYTE_ARRAY_BASE_OFFSET, - null, - dataBuffer.memoryAddress() + startIndex + totalBytes, - bytes.length); + bytes, + MemoryUtil.BYTE_ARRAY_BASE_OFFSET, + null, + dataBuffer.memoryAddress() + startIndex + totalBytes, + bytes.length); totalBytes += bytes.length; read += readSize; @@ -115,14 +105,10 @@ public void consume(ResultSet resultSet) throws SQLException { } } - /** - * Non-nullable consumer for clob data. - */ + /** Non-nullable consumer for clob data. */ static class NonNullableClobConsumer extends ClobConsumer { - /** - * Instantiate a ClobConsumer. - */ + /** Instantiate a ClobConsumer. */ public NonNullableClobConsumer(VarCharVector vector, int index) { super(vector, index); } @@ -148,11 +134,11 @@ public void consume(ResultSet resultSet) throws SQLException { vector.reallocDataBuffer(); } MemoryUtil.UNSAFE.copyMemory( - bytes, - MemoryUtil.BYTE_ARRAY_BASE_OFFSET, - null, - dataBuffer.memoryAddress() + startIndex + totalBytes, - bytes.length); + bytes, + MemoryUtil.BYTE_ARRAY_BASE_OFFSET, + null, + dataBuffer.memoryAddress() + startIndex + totalBytes, + bytes.length); totalBytes += bytes.length; read += readSize; @@ -161,7 +147,7 @@ public void consume(ResultSet resultSet) throws SQLException { BitVectorHelper.setBit(vector.getValidityBuffer(), currentIndex); vector.setLastSet(currentIndex); } - + currentIndex++; } } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/CompositeJdbcConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/CompositeJdbcConsumer.java index e6d780956d538..2366116fd0d18 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/CompositeJdbcConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/CompositeJdbcConsumer.java @@ -14,13 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.consumer; import java.io.IOException; import java.sql.ResultSet; import java.sql.SQLException; - import org.apache.arrow.adapter.jdbc.JdbcFieldInfo; import org.apache.arrow.adapter.jdbc.consumer.exceptions.JdbcConsumerException; import org.apache.arrow.util.AutoCloseables; @@ -28,17 +26,12 @@ import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.types.pojo.ArrowType; -/** - * Composite consumer which hold all consumers. - * It manages the consume and cleanup process. - */ +/** Composite consumer which hold all consumers. It manages the consume and cleanup process. */ public class CompositeJdbcConsumer implements JdbcConsumer { private final JdbcConsumer[] consumers; - /** - * Construct an instance. - */ + /** Construct an instance. */ public CompositeJdbcConsumer(JdbcConsumer[] consumers) { this.consumers = consumers; } @@ -51,9 +44,11 @@ public void consume(ResultSet rs) throws SQLException, IOException { } catch (Exception e) { if (consumers[i] instanceof BaseConsumer) { BaseConsumer consumer = (BaseConsumer) consumers[i]; - JdbcFieldInfo fieldInfo = new JdbcFieldInfo(rs.getMetaData(), consumer.columnIndexInResultSet); + JdbcFieldInfo fieldInfo = + new JdbcFieldInfo(rs.getMetaData(), consumer.columnIndexInResultSet); ArrowType arrowType = consumer.vector.getMinorType().getType(); - throw new JdbcConsumerException("Exception while consuming JDBC value", e, fieldInfo, arrowType); + throw new JdbcConsumerException( + "Exception while consuming JDBC value", e, fieldInfo, arrowType); } else { throw e; } @@ -70,17 +65,12 @@ public void close() { } catch (Exception e) { throw new RuntimeException("Error occurred while releasing resources.", e); } - } @Override - public void resetValueVector(ValueVector vector) { + public void resetValueVector(ValueVector vector) {} - } - - /** - * Reset inner consumers through vectors in the vector schema root. - */ + /** Reset inner consumers through vectors in the vector schema root. */ public void resetVectorSchemaRoot(VectorSchemaRoot root) { assert root.getFieldVectors().size() == consumers.length; for (int i = 0; i < consumers.length; i++) { @@ -88,4 +78,3 @@ public void resetVectorSchemaRoot(VectorSchemaRoot root) { } } } - diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DateConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DateConsumer.java index b9b83daccc25a..c271b900682a1 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DateConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DateConsumer.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.consumer; import java.sql.Date; @@ -22,19 +21,16 @@ import java.sql.SQLException; import java.util.Calendar; import java.util.concurrent.TimeUnit; - import org.apache.arrow.vector.DateDayVector; import org.apache.arrow.vector.DateMilliVector; /** - * Consumer which consume date type values from {@link ResultSet}. - * Write the data to {@link org.apache.arrow.vector.DateDayVector}. + * Consumer which consume date type values from {@link ResultSet}. Write the data to {@link + * org.apache.arrow.vector.DateDayVector}. */ public class DateConsumer { - /** - * Creates a consumer for {@link DateMilliVector}. - */ + /** Creates a consumer for {@link DateMilliVector}. */ public static JdbcConsumer createConsumer( DateDayVector vector, int index, boolean nullable, Calendar calendar) { if (nullable) { @@ -44,23 +40,17 @@ public static JdbcConsumer createConsumer( } } - /** - * Nullable consumer for date. - */ + /** Nullable consumer for date. */ static class NullableDateConsumer extends BaseConsumer { protected final Calendar calendar; - /** - * Instantiate a DateConsumer. - */ + /** Instantiate a DateConsumer. */ public NullableDateConsumer(DateDayVector vector, int index) { - this(vector, index, /* calendar */null); + this(vector, index, /* calendar */ null); } - /** - * Instantiate a DateConsumer. - */ + /** Instantiate a DateConsumer. */ public NullableDateConsumer(DateDayVector vector, int index, Calendar calendar) { super(vector, index); this.calendar = calendar; @@ -68,8 +58,10 @@ public NullableDateConsumer(DateDayVector vector, int index, Calendar calendar) @Override public void consume(ResultSet resultSet) throws SQLException { - Date date = calendar == null ? resultSet.getDate(columnIndexInResultSet) : - resultSet.getDate(columnIndexInResultSet, calendar); + Date date = + calendar == null + ? resultSet.getDate(columnIndexInResultSet) + : resultSet.getDate(columnIndexInResultSet, calendar); if (!resultSet.wasNull()) { // for fixed width vectors, we have allocated enough memory proactively, // so there is no need to call the setSafe method here. @@ -79,23 +71,17 @@ public void consume(ResultSet resultSet) throws SQLException { } } - /** - * Non-nullable consumer for date. - */ + /** Non-nullable consumer for date. */ static class NonNullableDateConsumer extends BaseConsumer { protected final Calendar calendar; - /** - * Instantiate a DateConsumer. - */ + /** Instantiate a DateConsumer. */ public NonNullableDateConsumer(DateDayVector vector, int index) { - this(vector, index, /* calendar */null); + this(vector, index, /* calendar */ null); } - /** - * Instantiate a DateConsumer. - */ + /** Instantiate a DateConsumer. */ public NonNullableDateConsumer(DateDayVector vector, int index, Calendar calendar) { super(vector, index); this.calendar = calendar; @@ -103,8 +89,10 @@ public NonNullableDateConsumer(DateDayVector vector, int index, Calendar calenda @Override public void consume(ResultSet resultSet) throws SQLException { - Date date = calendar == null ? resultSet.getDate(columnIndexInResultSet) : - resultSet.getDate(columnIndexInResultSet, calendar); + Date date = + calendar == null + ? resultSet.getDate(columnIndexInResultSet) + : resultSet.getDate(columnIndexInResultSet, calendar); // for fixed width vectors, we have allocated enough memory proactively, // so there is no need to call the setSafe method here. vector.set(currentIndex, Math.toIntExact(TimeUnit.MILLISECONDS.toDays(date.getTime()))); @@ -112,5 +100,3 @@ public void consume(ResultSet resultSet) throws SQLException { } } } - - diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/Decimal256Consumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/Decimal256Consumer.java index ad00d9b5a2492..eb33ea5038b98 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/Decimal256Consumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/Decimal256Consumer.java @@ -14,19 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.consumer; import java.math.BigDecimal; import java.math.RoundingMode; import java.sql.ResultSet; import java.sql.SQLException; - import org.apache.arrow.vector.Decimal256Vector; /** - * Consumer which consume decimal type values from {@link ResultSet}. - * Write the data to {@link org.apache.arrow.vector.Decimal256Vector}. + * Consumer which consume decimal type values from {@link ResultSet}. Write the data to {@link + * org.apache.arrow.vector.Decimal256Vector}. */ public abstract class Decimal256Consumer extends BaseConsumer { private final RoundingMode bigDecimalRoundingMode; @@ -36,7 +34,7 @@ public abstract class Decimal256Consumer extends BaseConsumer * Constructs a new consumer. * * @param vector the underlying vector for the consumer. - * @param index the column id for the consumer. + * @param index the column id for the consumer. */ public Decimal256Consumer(Decimal256Vector vector, int index) { this(vector, index, null); @@ -44,27 +42,23 @@ public Decimal256Consumer(Decimal256Vector vector, int index) { /** * Constructs a new consumer, with optional coercibility. + * * @param vector the underlying vector for the consumer. * @param index the column index for the consumer. - * @param bigDecimalRoundingMode java.math.RoundingMode to be applied if the BigDecimal scale does not match that - * of the target vector. Set to null to retain strict matching behavior (scale of - * source and target vector must match exactly). + * @param bigDecimalRoundingMode java.math.RoundingMode to be applied if the BigDecimal scale does + * not match that of the target vector. Set to null to retain strict matching behavior (scale + * of source and target vector must match exactly). */ - public Decimal256Consumer(Decimal256Vector vector, int index, RoundingMode bigDecimalRoundingMode) { + public Decimal256Consumer( + Decimal256Vector vector, int index, RoundingMode bigDecimalRoundingMode) { super(vector, index); this.bigDecimalRoundingMode = bigDecimalRoundingMode; this.scale = vector.getScale(); } - /** - * Creates a consumer for {@link Decimal256Vector}. - */ + /** Creates a consumer for {@link Decimal256Vector}. */ public static JdbcConsumer createConsumer( - Decimal256Vector vector, - int index, - boolean nullable, - RoundingMode bigDecimalRoundingMode - ) { + Decimal256Vector vector, int index, boolean nullable, RoundingMode bigDecimalRoundingMode) { if (nullable) { return new NullableDecimal256Consumer(vector, index, bigDecimalRoundingMode); } else { @@ -79,16 +73,12 @@ protected void set(BigDecimal value) { vector.set(currentIndex, value); } - - /** - * Consumer for nullable decimal. - */ + /** Consumer for nullable decimal. */ static class NullableDecimal256Consumer extends Decimal256Consumer { - /** - * Instantiate a Decimal256Consumer. - */ - public NullableDecimal256Consumer(Decimal256Vector vector, int index, RoundingMode bigDecimalRoundingMode) { + /** Instantiate a Decimal256Consumer. */ + public NullableDecimal256Consumer( + Decimal256Vector vector, int index, RoundingMode bigDecimalRoundingMode) { super(vector, index, bigDecimalRoundingMode); } @@ -104,15 +94,12 @@ public void consume(ResultSet resultSet) throws SQLException { } } - /** - * Consumer for non-nullable decimal. - */ + /** Consumer for non-nullable decimal. */ static class NonNullableDecimal256Consumer extends Decimal256Consumer { - /** - * Instantiate a Decimal256Consumer. - */ - public NonNullableDecimal256Consumer(Decimal256Vector vector, int index, RoundingMode bigDecimalRoundingMode) { + /** Instantiate a Decimal256Consumer. */ + public NonNullableDecimal256Consumer( + Decimal256Vector vector, int index, RoundingMode bigDecimalRoundingMode) { super(vector, index, bigDecimalRoundingMode); } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DecimalConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DecimalConsumer.java index bed96dda8b65d..05b4d27de1022 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DecimalConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DecimalConsumer.java @@ -14,19 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.consumer; import java.math.BigDecimal; import java.math.RoundingMode; import java.sql.ResultSet; import java.sql.SQLException; - import org.apache.arrow.vector.DecimalVector; /** - * Consumer which consume decimal type values from {@link ResultSet}. - * Write the data to {@link org.apache.arrow.vector.DecimalVector}. + * Consumer which consume decimal type values from {@link ResultSet}. Write the data to {@link + * org.apache.arrow.vector.DecimalVector}. */ public abstract class DecimalConsumer extends BaseConsumer { private final RoundingMode bigDecimalRoundingMode; @@ -36,7 +34,7 @@ public abstract class DecimalConsumer extends BaseConsumer { * Constructs a new consumer. * * @param vector the underlying vector for the consumer. - * @param index the column id for the consumer. + * @param index the column id for the consumer. */ public DecimalConsumer(DecimalVector vector, int index) { this(vector, index, null); @@ -44,11 +42,12 @@ public DecimalConsumer(DecimalVector vector, int index) { /** * Constructs a new consumer, with optional coercibility. + * * @param vector the underlying vector for the consumer. * @param index the column index for the consumer. - * @param bigDecimalRoundingMode java.math.RoundingMode to be applied if the BigDecimal scale does not match that - * of the target vector. Set to null to retain strict matching behavior (scale of - * source and target vector must match exactly). + * @param bigDecimalRoundingMode java.math.RoundingMode to be applied if the BigDecimal scale does + * not match that of the target vector. Set to null to retain strict matching behavior (scale + * of source and target vector must match exactly). */ public DecimalConsumer(DecimalVector vector, int index, RoundingMode bigDecimalRoundingMode) { super(vector, index); @@ -56,15 +55,9 @@ public DecimalConsumer(DecimalVector vector, int index, RoundingMode bigDecimalR this.scale = vector.getScale(); } - /** - * Creates a consumer for {@link DecimalVector}. - */ + /** Creates a consumer for {@link DecimalVector}. */ public static JdbcConsumer createConsumer( - DecimalVector vector, - int index, - boolean nullable, - RoundingMode bigDecimalRoundingMode - ) { + DecimalVector vector, int index, boolean nullable, RoundingMode bigDecimalRoundingMode) { if (nullable) { return new NullableDecimalConsumer(vector, index, bigDecimalRoundingMode); } else { @@ -79,16 +72,12 @@ protected void set(BigDecimal value) { vector.set(currentIndex, value); } - - /** - * Consumer for nullable decimal. - */ + /** Consumer for nullable decimal. */ static class NullableDecimalConsumer extends DecimalConsumer { - /** - * Instantiate a DecimalConsumer. - */ - public NullableDecimalConsumer(DecimalVector vector, int index, RoundingMode bigDecimalRoundingMode) { + /** Instantiate a DecimalConsumer. */ + public NullableDecimalConsumer( + DecimalVector vector, int index, RoundingMode bigDecimalRoundingMode) { super(vector, index, bigDecimalRoundingMode); } @@ -104,15 +93,12 @@ public void consume(ResultSet resultSet) throws SQLException { } } - /** - * Consumer for non-nullable decimal. - */ + /** Consumer for non-nullable decimal. */ static class NonNullableDecimalConsumer extends DecimalConsumer { - /** - * Instantiate a DecimalConsumer. - */ - public NonNullableDecimalConsumer(DecimalVector vector, int index, RoundingMode bigDecimalRoundingMode) { + /** Instantiate a DecimalConsumer. */ + public NonNullableDecimalConsumer( + DecimalVector vector, int index, RoundingMode bigDecimalRoundingMode) { super(vector, index, bigDecimalRoundingMode); } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DoubleConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DoubleConsumer.java index e3db95d1535af..9cd31e9245472 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DoubleConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/DoubleConsumer.java @@ -14,24 +14,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.consumer; import java.sql.ResultSet; import java.sql.SQLException; - import org.apache.arrow.vector.Float8Vector; /** - * Consumer which consume double type values from {@link ResultSet}. - * Write the data to {@link org.apache.arrow.vector.Float8Vector}. + * Consumer which consume double type values from {@link ResultSet}. Write the data to {@link + * org.apache.arrow.vector.Float8Vector}. */ public class DoubleConsumer { - /** - * Creates a consumer for {@link Float8Vector}. - */ - public static JdbcConsumer createConsumer(Float8Vector vector, int index, boolean nullable) { + /** Creates a consumer for {@link Float8Vector}. */ + public static JdbcConsumer createConsumer( + Float8Vector vector, int index, boolean nullable) { if (nullable) { return new NullableDoubleConsumer(vector, index); } else { @@ -39,14 +36,10 @@ public static JdbcConsumer createConsumer(Float8Vector vector, int } } - /** - * Nullable double consumer. - */ + /** Nullable double consumer. */ static class NullableDoubleConsumer extends BaseConsumer { - /** - * Instantiate a DoubleConsumer. - */ + /** Instantiate a DoubleConsumer. */ public NullableDoubleConsumer(Float8Vector vector, int index) { super(vector, index); } @@ -63,14 +56,10 @@ public void consume(ResultSet resultSet) throws SQLException { } } - /** - * Non-nullable double consumer. - */ + /** Non-nullable double consumer. */ static class NonNullableDoubleConsumer extends BaseConsumer { - /** - * Instantiate a DoubleConsumer. - */ + /** Instantiate a DoubleConsumer. */ public NonNullableDoubleConsumer(Float8Vector vector, int index) { super(vector, index); } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/FloatConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/FloatConsumer.java index 830348fe94c6b..0f16a68da883e 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/FloatConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/FloatConsumer.java @@ -14,24 +14,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.consumer; import java.sql.ResultSet; import java.sql.SQLException; - import org.apache.arrow.vector.Float4Vector; /** - * Consumer which consume float type values from {@link ResultSet}. - * Write the data to {@link org.apache.arrow.vector.Float4Vector}. + * Consumer which consume float type values from {@link ResultSet}. Write the data to {@link + * org.apache.arrow.vector.Float4Vector}. */ public class FloatConsumer { - /** - * Creates a consumer for {@link Float4Vector}. - */ - public static JdbcConsumer createConsumer(Float4Vector vector, int index, boolean nullable) { + /** Creates a consumer for {@link Float4Vector}. */ + public static JdbcConsumer createConsumer( + Float4Vector vector, int index, boolean nullable) { if (nullable) { return new NullableFloatConsumer(vector, index); } else { @@ -39,14 +36,10 @@ public static JdbcConsumer createConsumer(Float4Vector vector, int } } - /** - * Nullable float consumer. - */ + /** Nullable float consumer. */ static class NullableFloatConsumer extends BaseConsumer { - /** - * Instantiate a FloatConsumer. - */ + /** Instantiate a FloatConsumer. */ public NullableFloatConsumer(Float4Vector vector, int index) { super(vector, index); } @@ -63,14 +56,10 @@ public void consume(ResultSet resultSet) throws SQLException { } } - /** - * Non-nullable float consumer. - */ + /** Non-nullable float consumer. */ static class NonNullableFloatConsumer extends BaseConsumer { - /** - * Instantiate a FloatConsumer. - */ + /** Instantiate a FloatConsumer. */ public NonNullableFloatConsumer(Float4Vector vector, int index) { super(vector, index); } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/IntConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/IntConsumer.java index 4e537d682ff7c..302be697fbf07 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/IntConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/IntConsumer.java @@ -14,24 +14,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.consumer; import java.sql.ResultSet; import java.sql.SQLException; - import org.apache.arrow.vector.IntVector; /** - * Consumer which consume int type values from {@link ResultSet}. - * Write the data to {@link org.apache.arrow.vector.IntVector}. + * Consumer which consume int type values from {@link ResultSet}. Write the data to {@link + * org.apache.arrow.vector.IntVector}. */ public class IntConsumer { - /** - * Creates a consumer for {@link IntVector}. - */ - public static JdbcConsumer createConsumer(IntVector vector, int index, boolean nullable) { + /** Creates a consumer for {@link IntVector}. */ + public static JdbcConsumer createConsumer( + IntVector vector, int index, boolean nullable) { if (nullable) { return new NullableIntConsumer(vector, index); } else { @@ -39,14 +36,10 @@ public static JdbcConsumer createConsumer(IntVector vector, int index } } - /** - * Nullable consumer for int. - */ + /** Nullable consumer for int. */ static class NullableIntConsumer extends BaseConsumer { - /** - * Instantiate a IntConsumer. - */ + /** Instantiate a IntConsumer. */ public NullableIntConsumer(IntVector vector, int index) { super(vector, index); } @@ -63,14 +56,10 @@ public void consume(ResultSet resultSet) throws SQLException { } } - /** - * Non-nullable consumer for int. - */ + /** Non-nullable consumer for int. */ static class NonNullableIntConsumer extends BaseConsumer { - /** - * Instantiate a IntConsumer. - */ + /** Instantiate a IntConsumer. */ public NonNullableIntConsumer(IntVector vector, int index) { super(vector, index); } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/JdbcConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/JdbcConsumer.java index 7c867c7ad64d3..1ec6ad7eb9266 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/JdbcConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/JdbcConsumer.java @@ -14,34 +14,27 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.consumer; import java.io.IOException; import java.sql.ResultSet; import java.sql.SQLException; - import org.apache.arrow.vector.ValueVector; /** * An abstraction that is used to consume values from {@link ResultSet}. + * * @param The vector within consumer or its delegate, used for partially consume purpose. */ public interface JdbcConsumer extends AutoCloseable { - /** - * Consume a specific type value from {@link ResultSet} and write it to vector. - */ + /** Consume a specific type value from {@link ResultSet} and write it to vector. */ void consume(ResultSet resultSet) throws SQLException, IOException; - /** - * Close this consumer, do some clean work such as clear reuse ArrowBuf. - */ + /** Close this consumer, do some clean work such as clear reuse ArrowBuf. */ @Override void close() throws Exception; - /** - * Reset the vector within consumer for partial read purpose. - */ + /** Reset the vector within consumer for partial read purpose. */ void resetValueVector(T vector); } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/MapConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/MapConsumer.java index 07a071bfc096e..6223650ff2c04 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/MapConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/MapConsumer.java @@ -14,46 +14,39 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.consumer; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.sql.ResultSet; import java.sql.SQLException; import java.util.Map; - import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.impl.UnionMapWriter; import org.apache.arrow.vector.util.ObjectMapperFactory; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - /** - * Consumer which consume map type values from {@link ResultSet}. - * Write the data into {@link org.apache.arrow.vector.complex.MapVector}. + * Consumer which consume map type values from {@link ResultSet}. Write the data into {@link + * org.apache.arrow.vector.complex.MapVector}. */ public class MapConsumer extends BaseConsumer { - private final UnionMapWriter writer; private final ObjectMapper objectMapper = ObjectMapperFactory.newObjectMapper(); - private final TypeReference> typeReference = new TypeReference>() {}; + private final TypeReference> typeReference = + new TypeReference>() {}; private int currentRow; - /** - * Creates a consumer for {@link MapVector}. - */ + /** Creates a consumer for {@link MapVector}. */ public static MapConsumer createConsumer(MapVector mapVector, int index, boolean nullable) { return new MapConsumer(mapVector, index); } - /** - * Instantiate a MapConsumer. - */ + /** Instantiate a MapConsumer. */ public MapConsumer(MapVector vector, int index) { super(vector, index); writer = vector.getWriter(); @@ -69,7 +62,8 @@ public void consume(ResultSet resultSet) throws SQLException, IOException { } else if (map instanceof Map) { writeJavaMapIntoVector((Map) map); } else { - throw new IllegalArgumentException("Unknown type of map type column from JDBC " + map.getClass().getName()); + throw new IllegalArgumentException( + "Unknown type of map type column from JDBC " + map.getClass().getName()); } } else { writer.writeNull(); @@ -79,26 +73,25 @@ public void consume(ResultSet resultSet) throws SQLException, IOException { private void writeJavaMapIntoVector(Map map) { BufferAllocator allocator = vector.getAllocator(); writer.startMap(); - map.forEach((key, value) -> { - byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8); - byte[] valueBytes = value != null ? value.getBytes(StandardCharsets.UTF_8) : null; - try ( - ArrowBuf keyBuf = allocator.buffer(keyBytes.length); - ArrowBuf valueBuf = valueBytes != null ? allocator.buffer(valueBytes.length) : null; - ) { - writer.startEntry(); - keyBuf.writeBytes(keyBytes); - writer.key().varChar().writeVarChar(0, keyBytes.length, keyBuf); - if (valueBytes != null) { - valueBuf.writeBytes(valueBytes); - writer.value().varChar().writeVarChar(0, valueBytes.length, valueBuf); - } else { - writer.value().varChar().writeNull(); - } - writer.endEntry(); - } - }); + map.forEach( + (key, value) -> { + byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8); + byte[] valueBytes = value != null ? value.getBytes(StandardCharsets.UTF_8) : null; + try (ArrowBuf keyBuf = allocator.buffer(keyBytes.length); + ArrowBuf valueBuf = + valueBytes != null ? allocator.buffer(valueBytes.length) : null; ) { + writer.startEntry(); + keyBuf.writeBytes(keyBytes); + writer.key().varChar().writeVarChar(0, keyBytes.length, keyBuf); + if (valueBytes != null) { + valueBuf.writeBytes(valueBytes); + writer.value().varChar().writeVarChar(0, valueBytes.length, valueBuf); + } else { + writer.value().varChar().writeNull(); + } + writer.endEntry(); + } + }); writer.endMap(); } } - diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/NullConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/NullConsumer.java index a79a029f45d06..9d7a760f697a7 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/NullConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/NullConsumer.java @@ -14,17 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.consumer; import java.sql.ResultSet; import java.sql.SQLException; - import org.apache.arrow.vector.NullVector; /** - * Consumer which consume null type values from ResultSet. - * Corresponding to {@link org.apache.arrow.vector.NullVector}. + * Consumer which consume null type values from ResultSet. Corresponding to {@link + * org.apache.arrow.vector.NullVector}. */ public class NullConsumer extends BaseConsumer { @@ -33,6 +31,5 @@ public NullConsumer(NullVector vector) { } @Override - public void consume(ResultSet resultSet) throws SQLException { - } + public void consume(ResultSet resultSet) throws SQLException {} } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/SmallIntConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/SmallIntConsumer.java index 2edb3605b177a..9f45c077ed0a8 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/SmallIntConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/SmallIntConsumer.java @@ -14,24 +14,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.consumer; import java.sql.ResultSet; import java.sql.SQLException; - import org.apache.arrow.vector.SmallIntVector; /** - * Consumer which consume smallInt type values from {@link ResultSet}. - * Write the data to {@link org.apache.arrow.vector.SmallIntVector}. + * Consumer which consume smallInt type values from {@link ResultSet}. Write the data to {@link + * org.apache.arrow.vector.SmallIntVector}. */ public class SmallIntConsumer { - /** - * Creates a consumer for {@link SmallIntVector}. - */ - public static BaseConsumer createConsumer(SmallIntVector vector, int index, boolean nullable) { + /** Creates a consumer for {@link SmallIntVector}. */ + public static BaseConsumer createConsumer( + SmallIntVector vector, int index, boolean nullable) { if (nullable) { return new NullableSmallIntConsumer(vector, index); } else { @@ -39,14 +36,10 @@ public static BaseConsumer createConsumer(SmallIntVector vector, } } - /** - * Nullable consumer for small int. - */ + /** Nullable consumer for small int. */ static class NullableSmallIntConsumer extends BaseConsumer { - /** - * Instantiate a SmallIntConsumer. - */ + /** Instantiate a SmallIntConsumer. */ public NullableSmallIntConsumer(SmallIntVector vector, int index) { super(vector, index); } @@ -63,14 +56,10 @@ public void consume(ResultSet resultSet) throws SQLException { } } - /** - * Non-nullable consumer for small int. - */ + /** Non-nullable consumer for small int. */ static class NonNullableSmallIntConsumer extends BaseConsumer { - /** - * Instantiate a SmallIntConsumer. - */ + /** Instantiate a SmallIntConsumer. */ public NonNullableSmallIntConsumer(SmallIntVector vector, int index) { super(vector, index); } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TimeConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TimeConsumer.java index 4fa15ad79039e..bee19d0e4deab 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TimeConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TimeConsumer.java @@ -14,27 +14,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.consumer; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Time; import java.util.Calendar; - import org.apache.arrow.vector.TimeMilliVector; /** - * Consumer which consume time type values from {@link ResultSet}. - * Write the data to {@link org.apache.arrow.vector.TimeMilliVector}. + * Consumer which consume time type values from {@link ResultSet}. Write the data to {@link + * org.apache.arrow.vector.TimeMilliVector}. */ public abstract class TimeConsumer { - /** - * Creates a consumer for {@link TimeMilliVector}. - */ + /** Creates a consumer for {@link TimeMilliVector}. */ public static JdbcConsumer createConsumer( - TimeMilliVector vector, int index, boolean nullable, Calendar calendar) { + TimeMilliVector vector, int index, boolean nullable, Calendar calendar) { if (nullable) { return new NullableTimeConsumer(vector, index, calendar); } else { @@ -42,23 +38,17 @@ public static JdbcConsumer createConsumer( } } - /** - * Nullable consumer for {@link TimeMilliVector}. - */ + /** Nullable consumer for {@link TimeMilliVector}. */ static class NullableTimeConsumer extends BaseConsumer { protected final Calendar calendar; - /** - * Instantiate a TimeConsumer. - */ + /** Instantiate a TimeConsumer. */ public NullableTimeConsumer(TimeMilliVector vector, int index) { - this(vector, index, /* calendar */null); + this(vector, index, /* calendar */ null); } - /** - * Instantiate a TimeConsumer. - */ + /** Instantiate a TimeConsumer. */ public NullableTimeConsumer(TimeMilliVector vector, int index, Calendar calendar) { super(vector, index); this.calendar = calendar; @@ -66,8 +56,10 @@ public NullableTimeConsumer(TimeMilliVector vector, int index, Calendar calendar @Override public void consume(ResultSet resultSet) throws SQLException { - Time time = calendar == null ? resultSet.getTime(columnIndexInResultSet) : - resultSet.getTime(columnIndexInResultSet, calendar); + Time time = + calendar == null + ? resultSet.getTime(columnIndexInResultSet) + : resultSet.getTime(columnIndexInResultSet, calendar); if (!resultSet.wasNull()) { // for fixed width vectors, we have allocated enough memory proactively, // so there is no need to call the setSafe method here. @@ -77,23 +69,17 @@ public void consume(ResultSet resultSet) throws SQLException { } } - /** - * Non-nullable consumer for {@link TimeMilliVector}. - */ + /** Non-nullable consumer for {@link TimeMilliVector}. */ static class NonNullableTimeConsumer extends BaseConsumer { protected final Calendar calendar; - /** - * Instantiate a TimeConsumer. - */ + /** Instantiate a TimeConsumer. */ public NonNullableTimeConsumer(TimeMilliVector vector, int index) { - this(vector, index, /* calendar */null); + this(vector, index, /* calendar */ null); } - /** - * Instantiate a TimeConsumer. - */ + /** Instantiate a TimeConsumer. */ public NonNullableTimeConsumer(TimeMilliVector vector, int index, Calendar calendar) { super(vector, index); this.calendar = calendar; @@ -101,8 +87,10 @@ public NonNullableTimeConsumer(TimeMilliVector vector, int index, Calendar calen @Override public void consume(ResultSet resultSet) throws SQLException { - Time time = calendar == null ? resultSet.getTime(columnIndexInResultSet) : - resultSet.getTime(columnIndexInResultSet, calendar); + Time time = + calendar == null + ? resultSet.getTime(columnIndexInResultSet) + : resultSet.getTime(columnIndexInResultSet, calendar); // for fixed width vectors, we have allocated enough memory proactively, // so there is no need to call the setSafe method here. vector.set(currentIndex, (int) time.getTime()); diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TimestampConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TimestampConsumer.java index 3351e7e78a7e4..cc6269c21f04a 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TimestampConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TimestampConsumer.java @@ -14,26 +14,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.consumer; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Timestamp; - import org.apache.arrow.vector.TimeStampMilliVector; /** - * Consumer which consume timestamp type values from {@link ResultSet}. - * Write the data to {@link TimeStampMilliVector}. + * Consumer which consume timestamp type values from {@link ResultSet}. Write the data to {@link + * TimeStampMilliVector}. */ public abstract class TimestampConsumer { - /** - * Creates a consumer for {@link TimeStampMilliVector}. - */ + /** Creates a consumer for {@link TimeStampMilliVector}. */ public static JdbcConsumer createConsumer( - TimeStampMilliVector vector, int index, boolean nullable) { + TimeStampMilliVector vector, int index, boolean nullable) { if (nullable) { return new NullableTimestampConsumer(vector, index); } else { @@ -41,14 +37,10 @@ public static JdbcConsumer createConsumer( } } - /** - * Nullable consumer for timestamp. - */ + /** Nullable consumer for timestamp. */ static class NullableTimestampConsumer extends BaseConsumer { - /** - * Instantiate a TimestampConsumer. - */ + /** Instantiate a TimestampConsumer. */ public NullableTimestampConsumer(TimeStampMilliVector vector, int index) { super(vector, index); } @@ -65,14 +57,10 @@ public void consume(ResultSet resultSet) throws SQLException { } } - /** - * Non-nullable consumer for timestamp. - */ + /** Non-nullable consumer for timestamp. */ static class NonNullableTimestampConsumer extends BaseConsumer { - /** - * Instantiate a TimestampConsumer. - */ + /** Instantiate a TimestampConsumer. */ public NonNullableTimestampConsumer(TimeStampMilliVector vector, int index) { super(vector, index); } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TimestampTZConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TimestampTZConsumer.java index f08671f0be61a..3e4911ac1a161 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TimestampTZConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TimestampTZConsumer.java @@ -14,25 +14,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.consumer; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Timestamp; import java.util.Calendar; - import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.TimeStampMilliTZVector; /** - * Consumer which consume timestamp (with time zone) type values from {@link ResultSet}. - * Write the data to {@link TimeStampMilliTZVector}. + * Consumer which consume timestamp (with time zone) type values from {@link ResultSet}. Write the + * data to {@link TimeStampMilliTZVector}. */ public class TimestampTZConsumer { - /** - * Creates a consumer for {@link TimeStampMilliTZVector}. - */ + /** Creates a consumer for {@link TimeStampMilliTZVector}. */ public static JdbcConsumer createConsumer( TimeStampMilliTZVector vector, int index, boolean nullable, Calendar calendar) { Preconditions.checkArgument(calendar != null, "Calendar cannot be null"); @@ -43,17 +39,14 @@ public static JdbcConsumer createConsumer( } } - /** - * Nullable consumer for timestamp (with time zone). - */ + /** Nullable consumer for timestamp (with time zone). */ static class NullableTimestampTZConsumer extends BaseConsumer { protected final Calendar calendar; - /** - * Instantiate a TimestampConsumer. - */ - public NullableTimestampTZConsumer(TimeStampMilliTZVector vector, int index, Calendar calendar) { + /** Instantiate a TimestampConsumer. */ + public NullableTimestampTZConsumer( + TimeStampMilliTZVector vector, int index, Calendar calendar) { super(vector, index); this.calendar = calendar; } @@ -70,17 +63,14 @@ public void consume(ResultSet resultSet) throws SQLException { } } - /** - * Non-nullable consumer for timestamp (with time zone). - */ + /** Non-nullable consumer for timestamp (with time zone). */ static class NonNullableTimestampConsumer extends BaseConsumer { protected final Calendar calendar; - /** - * Instantiate a TimestampConsumer. - */ - public NonNullableTimestampConsumer(TimeStampMilliTZVector vector, int index, Calendar calendar) { + /** Instantiate a TimestampConsumer. */ + public NonNullableTimestampConsumer( + TimeStampMilliTZVector vector, int index, Calendar calendar) { super(vector, index); this.calendar = calendar; } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TinyIntConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TinyIntConsumer.java index 40cf087a5ec66..b75b87dd81cc4 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TinyIntConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/TinyIntConsumer.java @@ -14,24 +14,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.consumer; import java.sql.ResultSet; import java.sql.SQLException; - import org.apache.arrow.vector.TinyIntVector; /** - * Consumer which consume tinyInt type values from {@link ResultSet}. - * Write the data to {@link org.apache.arrow.vector.TinyIntVector}. + * Consumer which consume tinyInt type values from {@link ResultSet}. Write the data to {@link + * org.apache.arrow.vector.TinyIntVector}. */ public abstract class TinyIntConsumer { - /** - * Creates a consumer for {@link TinyIntVector}. - */ - public static JdbcConsumer createConsumer(TinyIntVector vector, int index, boolean nullable) { + /** Creates a consumer for {@link TinyIntVector}. */ + public static JdbcConsumer createConsumer( + TinyIntVector vector, int index, boolean nullable) { if (nullable) { return new NullableTinyIntConsumer(vector, index); } else { @@ -39,14 +36,10 @@ public static JdbcConsumer createConsumer(TinyIntVector vector, i } } - /** - * Nullable consumer for tiny int. - */ + /** Nullable consumer for tiny int. */ static class NullableTinyIntConsumer extends BaseConsumer { - /** - * Instantiate a TinyIntConsumer. - */ + /** Instantiate a TinyIntConsumer. */ public NullableTinyIntConsumer(TinyIntVector vector, int index) { super(vector, index); } @@ -63,14 +56,10 @@ public void consume(ResultSet resultSet) throws SQLException { } } - /** - * Non-nullable consumer for tiny int. - */ + /** Non-nullable consumer for tiny int. */ static class NonNullableTinyIntConsumer extends BaseConsumer { - /** - * Instantiate a TinyIntConsumer. - */ + /** Instantiate a TinyIntConsumer. */ public NonNullableTinyIntConsumer(TinyIntVector vector, int index) { super(vector, index); } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/VarCharConsumer.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/VarCharConsumer.java index 05333715b8c2f..c81c4f0db124b 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/VarCharConsumer.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/VarCharConsumer.java @@ -14,25 +14,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.consumer; import java.nio.charset.StandardCharsets; import java.sql.ResultSet; import java.sql.SQLException; - import org.apache.arrow.vector.VarCharVector; /** - * Consumer which consume varchar type values from {@link ResultSet}. - * Write the data to {@link org.apache.arrow.vector.VarCharVector}. + * Consumer which consume varchar type values from {@link ResultSet}. Write the data to {@link + * org.apache.arrow.vector.VarCharVector}. */ public abstract class VarCharConsumer { - /** - * Creates a consumer for {@link VarCharVector}. - */ - public static JdbcConsumer createConsumer(VarCharVector vector, int index, boolean nullable) { + /** Creates a consumer for {@link VarCharVector}. */ + public static JdbcConsumer createConsumer( + VarCharVector vector, int index, boolean nullable) { if (nullable) { return new NullableVarCharConsumer(vector, index); } else { @@ -40,14 +37,10 @@ public static JdbcConsumer createConsumer(VarCharVector vector, i } } - /** - * Nullable consumer for var char. - */ + /** Nullable consumer for var char. */ static class NullableVarCharConsumer extends BaseConsumer { - /** - * Instantiate a VarCharConsumer. - */ + /** Instantiate a VarCharConsumer. */ public NullableVarCharConsumer(VarCharVector vector, int index) { super(vector, index); } @@ -63,14 +56,10 @@ public void consume(ResultSet resultSet) throws SQLException { } } - /** - * Non-nullable consumer for var char. - */ + /** Non-nullable consumer for var char. */ static class NonNullableVarCharConsumer extends BaseConsumer { - /** - * Instantiate a VarCharConsumer. - */ + /** Instantiate a VarCharConsumer. */ public NonNullableVarCharConsumer(VarCharVector vector, int index) { super(vector, index); } diff --git a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/exceptions/JdbcConsumerException.java b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/exceptions/JdbcConsumerException.java index b235be173cf10..04e26d640c04d 100644 --- a/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/exceptions/JdbcConsumerException.java +++ b/java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/consumer/exceptions/JdbcConsumerException.java @@ -14,15 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.consumer.exceptions; import org.apache.arrow.adapter.jdbc.JdbcFieldInfo; import org.apache.arrow.vector.types.pojo.ArrowType; /** - * Exception while consuming JDBC data. This exception stores the JdbcFieldInfo for the column and the - * ArrowType for the corresponding vector for easier debugging. + * Exception while consuming JDBC data. This exception stores the JdbcFieldInfo for the column and + * the ArrowType for the corresponding vector for easier debugging. */ public class JdbcConsumerException extends RuntimeException { final JdbcFieldInfo fieldInfo; @@ -31,12 +30,13 @@ public class JdbcConsumerException extends RuntimeException { /** * Construct JdbcConsumerException with all fields. * - * @param message error message - * @param cause original exception + * @param message error message + * @param cause original exception * @param fieldInfo JdbcFieldInfo for the column * @param arrowType ArrowType for the corresponding vector */ - public JdbcConsumerException(String message, Throwable cause, JdbcFieldInfo fieldInfo, ArrowType arrowType) { + public JdbcConsumerException( + String message, Throwable cause, JdbcFieldInfo fieldInfo, ArrowType arrowType) { super(message, cause); this.fieldInfo = fieldInfo; this.arrowType = arrowType; diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java index 88a66a31aa2c9..1ad4492b35d18 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/AbstractJdbcToArrowTest.java @@ -14,9 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; @@ -30,7 +31,6 @@ import java.util.Map; import java.util.TimeZone; import java.util.function.Function; - import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.util.Preconditions; @@ -41,12 +41,7 @@ import org.junit.Before; import org.junit.Test; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; - -/** - * Class to abstract out some common test functionality for testing JDBC to Arrow. - */ +/** Class to abstract out some common test functionality for testing JDBC to Arrow. */ public abstract class AbstractJdbcToArrowTest { protected static final String BIGINT = "BIGINT_FIELD5"; @@ -69,7 +64,8 @@ public abstract class AbstractJdbcToArrowTest { protected static final String TINYINT = "TINYINT_FIELD3"; protected static final String VARCHAR = "VARCHAR_FIELD13"; protected static final String NULL = "NULL_FIELD18"; - protected static final Map ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP = new HashMap<>(); + protected static final Map ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP = + new HashMap<>(); static { ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP.put(LIST, new JdbcFieldInfo(Types.INTEGER)); @@ -86,12 +82,12 @@ public abstract class AbstractJdbcToArrowTest { * @return Table object * @throws IOException on error */ - protected static Table getTable(String ymlFilePath, @SuppressWarnings("rawtypes") Class clss) throws IOException { - return new ObjectMapper(new YAMLFactory()).readValue( - clss.getClassLoader().getResourceAsStream(ymlFilePath), Table.class); + protected static Table getTable(String ymlFilePath, @SuppressWarnings("rawtypes") Class clss) + throws IOException { + return new ObjectMapper(new YAMLFactory()) + .readValue(clss.getClassLoader().getResourceAsStream(ymlFilePath), Table.class); } - /** * This method creates Connection object and DB table and also populate data into table for test. * @@ -105,7 +101,7 @@ public void setUp() throws SQLException, ClassNotFoundException { String driver = "org.h2.Driver"; Class.forName(driver); conn = DriverManager.getConnection(url); - try (Statement stmt = conn.createStatement();) { + try (Statement stmt = conn.createStatement(); ) { stmt.executeUpdate(table.getCreate()); for (String insert : table.getData()) { stmt.executeUpdate(insert); @@ -136,12 +132,13 @@ public void destroy() throws SQLException { * @throws ClassNotFoundException on error * @throws IOException on error */ - public static Object[][] prepareTestData(String[] testFiles, @SuppressWarnings("rawtypes") Class clss) + public static Object[][] prepareTestData( + String[] testFiles, @SuppressWarnings("rawtypes") Class clss) throws SQLException, ClassNotFoundException, IOException { Object[][] tableArr = new Object[testFiles.length][]; int i = 0; for (String testFile : testFiles) { - tableArr[i++] = new Object[]{getTable(testFile, clss)}; + tableArr[i++] = new Object[] {getTable(testFile, clss)}; } return tableArr; } @@ -159,86 +156,90 @@ public static Object[][] prepareTestData(String[] testFiles, @SuppressWarnings(" * Abstract method to implement logic to assert test various datatype values. * * @param root VectorSchemaRoot for test - * @param isIncludeMapVector is this dataset checks includes map column. - * Jdbc type to 'map' mapping declared in configuration only manually + * @param isIncludeMapVector is this dataset checks includes map column. Jdbc type to 'map' + * mapping declared in configuration only manually */ public abstract void testDataSets(VectorSchemaRoot root, boolean isIncludeMapVector); /** - * For the given SQL query, execute and fetch the data from Relational DB and convert it to Arrow objects. - * This method uses the default Calendar instance with default TimeZone and Locale as returned by the JVM. - * If you wish to use specific TimeZone or Locale for any Date, Time and Timestamp datasets, you may want use - * overloaded API that taken Calendar object instance. + * For the given SQL query, execute and fetch the data from Relational DB and convert it to Arrow + * objects. This method uses the default Calendar instance with default TimeZone and Locale as + * returned by the JVM. If you wish to use specific TimeZone or Locale for any Date, Time and + * Timestamp datasets, you may want use overloaded API that taken Calendar object instance. * - * This method is for test only. + *

    This method is for test only. * - * @param connection Database connection to be used. This method will not close the passed connection object. Since - * the caller has passed the connection object it's the responsibility of the caller to close or - * return the connection to the pool. - * @param query The DB Query to fetch the data. - * @param allocator Memory allocator + * @param connection Database connection to be used. This method will not close the passed + * connection object. Since the caller has passed the connection object it's the + * responsibility of the caller to close or return the connection to the pool. + * @param query The DB Query to fetch the data. + * @param allocator Memory allocator * @return Arrow Data Objects {@link VectorSchemaRoot} - * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources opened such as - * ResultSet and Statement objects. + * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources + * opened such as ResultSet and Statement objects. */ public VectorSchemaRoot sqlToArrow(Connection connection, String query, BufferAllocator allocator) throws SQLException, IOException { Preconditions.checkNotNull(allocator, "Memory allocator object cannot be null"); - JdbcToArrowConfig config = new JdbcToArrowConfigBuilder(allocator, JdbcToArrowUtils.getUtcCalendar()) - .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) - .build(); + JdbcToArrowConfig config = + new JdbcToArrowConfigBuilder(allocator, JdbcToArrowUtils.getUtcCalendar()) + .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) + .build(); return sqlToArrow(connection, query, config); } /** - * For the given SQL query, execute and fetch the data from Relational DB and convert it to Arrow objects. + * For the given SQL query, execute and fetch the data from Relational DB and convert it to Arrow + * objects. * - * This method is for test only. + *

    This method is for test only. * - * @param connection Database connection to be used. This method will not close the passed connection object. Since - * the caller has passed the connection object it's the responsibility of the caller to close or - * return the connection to the pool. - * @param query The DB Query to fetch the data. - * @param allocator Memory allocator - * @param calendar Calendar object to use to handle Date, Time and Timestamp datasets. + * @param connection Database connection to be used. This method will not close the passed + * connection object. Since the caller has passed the connection object it's the + * responsibility of the caller to close or return the connection to the pool. + * @param query The DB Query to fetch the data. + * @param allocator Memory allocator + * @param calendar Calendar object to use to handle Date, Time and Timestamp datasets. * @return Arrow Data Objects {@link VectorSchemaRoot} - * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources opened such as - * ResultSet and Statement objects. + * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources + * opened such as ResultSet and Statement objects. */ public VectorSchemaRoot sqlToArrow( - Connection connection, - String query, - BufferAllocator allocator, - Calendar calendar) throws SQLException, IOException { + Connection connection, String query, BufferAllocator allocator, Calendar calendar) + throws SQLException, IOException { Preconditions.checkNotNull(allocator, "Memory allocator object cannot be null"); Preconditions.checkNotNull(calendar, "Calendar object cannot be null"); - JdbcToArrowConfig config = new JdbcToArrowConfigBuilder(allocator, calendar) - .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) - .build(); + JdbcToArrowConfig config = + new JdbcToArrowConfigBuilder(allocator, calendar) + .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) + .build(); return sqlToArrow(connection, query, config); } /** - * For the given SQL query, execute and fetch the data from Relational DB and convert it to Arrow objects. + * For the given SQL query, execute and fetch the data from Relational DB and convert it to Arrow + * objects. * - * This method is for test only. + *

    This method is for test only. * - * @param connection Database connection to be used. This method will not close the passed connection object. - * Since the caller has passed the connection object it's the responsibility of the caller - * to close or return the connection to the pool. - * @param query The DB Query to fetch the data. - * @param config Configuration + * @param connection Database connection to be used. This method will not close the passed + * connection object. Since the caller has passed the connection object it's the + * responsibility of the caller to close or return the connection to the pool. + * @param query The DB Query to fetch the data. + * @param config Configuration * @return Arrow Data Objects {@link VectorSchemaRoot} - * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources opened such as - * ResultSet and Statement objects. + * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources + * opened such as ResultSet and Statement objects. */ - public static VectorSchemaRoot sqlToArrow(Connection connection, String query, JdbcToArrowConfig config) + public static VectorSchemaRoot sqlToArrow( + Connection connection, String query, JdbcToArrowConfig config) throws SQLException, IOException { Preconditions.checkNotNull(connection, "JDBC connection object cannot be null"); - Preconditions.checkArgument(query != null && query.length() > 0, "SQL query cannot be null or empty"); + Preconditions.checkArgument( + query != null && query.length() > 0, "SQL query cannot be null or empty"); try (Statement stmt = connection.createStatement()) { return sqlToArrow(stmt.executeQuery(query), config); @@ -246,10 +247,10 @@ public static VectorSchemaRoot sqlToArrow(Connection connection, String query, J } /** - * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. This - * method uses the default RootAllocator and Calendar object. + * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow + * objects. This method uses the default RootAllocator and Calendar object. * - * This method is for test only. + *

    This method is for test only. * * @param resultSet ResultSet to use to fetch the data from underlying database * @return Arrow Data Objects {@link VectorSchemaRoot} @@ -262,9 +263,10 @@ public static VectorSchemaRoot sqlToArrow(ResultSet resultSet) throws SQLExcepti } /** - * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. + * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow + * objects. * - * This method is for test only. + *

    This method is for test only. * * @param resultSet ResultSet to use to fetch the data from underlying database * @param allocator Memory allocator @@ -275,62 +277,69 @@ public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, BufferAllocator a throws SQLException, IOException { Preconditions.checkNotNull(allocator, "Memory Allocator object cannot be null"); - JdbcToArrowConfig config = new JdbcToArrowConfigBuilder(allocator, JdbcToArrowUtils.getUtcCalendar()) - .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) - .build(); + JdbcToArrowConfig config = + new JdbcToArrowConfigBuilder(allocator, JdbcToArrowUtils.getUtcCalendar()) + .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) + .build(); return sqlToArrow(resultSet, config); } /** - * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. + * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow + * objects. * - * This method is for test only. + *

    This method is for test only. * * @param resultSet ResultSet to use to fetch the data from underlying database - * @param calendar Calendar instance to use for Date, Time and Timestamp datasets, or null if none. + * @param calendar Calendar instance to use for Date, Time and Timestamp datasets, or null + * if none. * @return Arrow Data Objects {@link VectorSchemaRoot} * @throws SQLException on error */ - public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, Calendar calendar) throws SQLException, IOException { + public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, Calendar calendar) + throws SQLException, IOException { Preconditions.checkNotNull(resultSet, "JDBC ResultSet object cannot be null"); - JdbcToArrowConfig config = new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), calendar) - .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) - .build(); + JdbcToArrowConfig config = + new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), calendar) + .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) + .build(); return sqlToArrow(resultSet, config); } /** - * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. + * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow + * objects. * - * This method is for test only. + *

    This method is for test only. * * @param resultSet ResultSet to use to fetch the data from underlying database * @param allocator Memory allocator to use. - * @param calendar Calendar instance to use for Date, Time and Timestamp datasets, or null if none. + * @param calendar Calendar instance to use for Date, Time and Timestamp datasets, or null + * if none. * @return Arrow Data Objects {@link VectorSchemaRoot} * @throws SQLException on error */ public static VectorSchemaRoot sqlToArrow( - ResultSet resultSet, - BufferAllocator allocator, - Calendar calendar) + ResultSet resultSet, BufferAllocator allocator, Calendar calendar) throws SQLException, IOException { Preconditions.checkNotNull(allocator, "Memory Allocator object cannot be null"); - JdbcToArrowConfig config = new JdbcToArrowConfigBuilder(allocator, calendar) - .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) - .build(); + JdbcToArrowConfig config = + new JdbcToArrowConfigBuilder(allocator, calendar) + .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) + .build(); return sqlToArrow(resultSet, config); } /** - * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow objects. + * For the given JDBC {@link ResultSet}, fetch the data from Relational DB and convert it to Arrow + * objects. * - * This method is for test only. + *

    This method is for test only. * * @param resultSet ResultSet to use to fetch the data from underlying database - * @param config Configuration of the conversion from JDBC to Arrow. + * @param config Configuration of the conversion from JDBC to Arrow. * @return Arrow Data Objects {@link VectorSchemaRoot} * @throws SQLException on error */ @@ -339,8 +348,10 @@ public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, JdbcToArrowConfig Preconditions.checkNotNull(resultSet, "JDBC ResultSet object cannot be null"); Preconditions.checkNotNull(config, "The configuration cannot be null"); - VectorSchemaRoot root = VectorSchemaRoot.create( - JdbcToArrowUtils.jdbcToArrowSchema(resultSet.getMetaData(), config), config.getAllocator()); + VectorSchemaRoot root = + VectorSchemaRoot.create( + JdbcToArrowUtils.jdbcToArrowSchema(resultSet.getMetaData(), config), + config.getAllocator()); if (config.getTargetBatchSize() != JdbcToArrowConfig.NO_LIMIT_BATCH_SIZE) { ValueVectorUtility.preAllocate(root, config.getTargetBatchSize()); } @@ -350,12 +361,14 @@ public static VectorSchemaRoot sqlToArrow(ResultSet resultSet, JdbcToArrowConfig /** * Register MAP_FIELD20 as ArrowType.Map - * @param calendar Calendar instance to use for Date, Time and Timestamp datasets, or null if none. + * + * @param calendar Calendar instance to use for Date, Time and Timestamp datasets, or null + * if none. * @param rsmd ResultSetMetaData to lookup column name from result set metadata * @return typeConverter instance with mapping column to Map type */ protected Function jdbcToArrowTypeConverter( - Calendar calendar, ResultSetMetaData rsmd) { + Calendar calendar, ResultSetMetaData rsmd) { return (jdbcFieldInfo) -> { String columnLabel = null; try { @@ -377,5 +390,4 @@ protected Function jdbcToArrowTypeConverter( protected ResultSetMetaData getQueryMetaData(String query) throws SQLException { return conn.createStatement().executeQuery(query).getMetaData(); } - } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcFieldInfoTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcFieldInfoTest.java index b1a8b8f226753..cd6a78eae2b1a 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcFieldInfoTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcFieldInfoTest.java @@ -14,13 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc; import static org.junit.Assert.*; import java.sql.Types; - import org.junit.Test; public class JdbcFieldInfoTest { diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcParameterBinderTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcParameterBinderTest.java index a94f0aa454f1d..a05130f18e4ac 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcParameterBinderTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcParameterBinderTest.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc; import static org.assertj.core.api.Assertions.assertThat; @@ -32,7 +31,6 @@ import java.util.List; import java.util.Map; import java.util.function.BiConsumer; - import org.apache.arrow.adapter.jdbc.binder.ColumnBinder; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; @@ -106,7 +104,7 @@ void bindOrder() throws SQLException { Field.nullable("ints1", new ArrowType.Int(32, true)), Field.nullable("ints2", new ArrowType.Int(32, true)))); try (final MockPreparedStatement statement = new MockPreparedStatement(); - final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { + final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { final JdbcParameterBinder binder = JdbcParameterBinder.builder(statement, root) .bind(/*parameterIndex=*/ 1, /*columnIndex=*/ 2) @@ -161,17 +159,17 @@ void bindOrder() throws SQLException { @Test void customBinder() throws SQLException { final Schema schema = - new Schema(Collections.singletonList( - Field.nullable("ints0", new ArrowType.Int(32, true)))); + new Schema(Collections.singletonList(Field.nullable("ints0", new ArrowType.Int(32, true)))); try (final MockPreparedStatement statement = new MockPreparedStatement(); - final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { + final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { final JdbcParameterBinder binder = JdbcParameterBinder.builder(statement, root) .bind( /*parameterIndex=*/ 1, new ColumnBinder() { private final IntVector vector = (IntVector) root.getVector(0); + @Override public void bind(PreparedStatement statement, int parameterIndex, int rowIndex) throws SQLException { @@ -212,7 +210,9 @@ public FieldVector getVector() { @Test void bool() throws SQLException { - testSimpleType(ArrowType.Bool.INSTANCE, Types.BOOLEAN, + testSimpleType( + ArrowType.Bool.INSTANCE, + Types.BOOLEAN, (BitVector vector, Integer index, Boolean value) -> vector.setSafe(index, value ? 1 : 0), BitVector::setNull, Arrays.asList(true, false, true)); @@ -220,53 +220,76 @@ void bool() throws SQLException { @Test void int8() throws SQLException { - testSimpleType(new ArrowType.Int(8, true), Types.TINYINT, - TinyIntVector::setSafe, TinyIntVector::setNull, + testSimpleType( + new ArrowType.Int(8, true), + Types.TINYINT, + TinyIntVector::setSafe, + TinyIntVector::setNull, Arrays.asList(Byte.MAX_VALUE, Byte.MIN_VALUE, (byte) 42)); } @Test void int16() throws SQLException { - testSimpleType(new ArrowType.Int(16, true), Types.SMALLINT, - SmallIntVector::setSafe, SmallIntVector::setNull, + testSimpleType( + new ArrowType.Int(16, true), + Types.SMALLINT, + SmallIntVector::setSafe, + SmallIntVector::setNull, Arrays.asList(Short.MAX_VALUE, Short.MIN_VALUE, (short) 42)); } @Test void int32() throws SQLException { - testSimpleType(new ArrowType.Int(32, true), Types.INTEGER, - IntVector::setSafe, IntVector::setNull, + testSimpleType( + new ArrowType.Int(32, true), + Types.INTEGER, + IntVector::setSafe, + IntVector::setNull, Arrays.asList(Integer.MAX_VALUE, Integer.MIN_VALUE, 42)); } @Test void int64() throws SQLException { - testSimpleType(new ArrowType.Int(64, true), Types.BIGINT, - BigIntVector::setSafe, BigIntVector::setNull, + testSimpleType( + new ArrowType.Int(64, true), + Types.BIGINT, + BigIntVector::setSafe, + BigIntVector::setNull, Arrays.asList(Long.MAX_VALUE, Long.MIN_VALUE, 42L)); } @Test void float32() throws SQLException { - testSimpleType(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE), Types.REAL, - Float4Vector::setSafe, Float4Vector::setNull, + testSimpleType( + new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE), + Types.REAL, + Float4Vector::setSafe, + Float4Vector::setNull, Arrays.asList(Float.MIN_VALUE, Float.MAX_VALUE, Float.POSITIVE_INFINITY)); } @Test void float64() throws SQLException { - testSimpleType(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE), Types.DOUBLE, - Float8Vector::setSafe, Float8Vector::setNull, + testSimpleType( + new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE), + Types.DOUBLE, + Float8Vector::setSafe, + Float8Vector::setNull, Arrays.asList(Double.MIN_VALUE, Double.MAX_VALUE, Double.POSITIVE_INFINITY)); } @Test void time32() throws SQLException { - testSimpleType(new ArrowType.Time(TimeUnit.SECOND, 32), Types.TIME, - (valueVectors, index, value) -> valueVectors.setSafe(index, (int) (value.getTime() / 1_000)), + testSimpleType( + new ArrowType.Time(TimeUnit.SECOND, 32), + Types.TIME, + (valueVectors, index, value) -> + valueVectors.setSafe(index, (int) (value.getTime() / 1_000)), TimeSecVector::setNull, Arrays.asList(new Time(-128_000), new Time(104_000), new Time(-42_000))); - testSimpleType(new ArrowType.Time(TimeUnit.MILLISECOND, 32), Types.TIME, + testSimpleType( + new ArrowType.Time(TimeUnit.MILLISECOND, 32), + Types.TIME, (valueVectors, index, value) -> valueVectors.setSafe(index, (int) value.getTime()), TimeMilliVector::setNull, Arrays.asList(new Time(-128_000), new Time(104_000), new Time(-42_000))); @@ -274,11 +297,15 @@ void time32() throws SQLException { @Test void time64() throws SQLException { - testSimpleType(new ArrowType.Time(TimeUnit.MICROSECOND, 64), Types.TIME, + testSimpleType( + new ArrowType.Time(TimeUnit.MICROSECOND, 64), + Types.TIME, (valueVectors, index, value) -> valueVectors.setSafe(index, (value.getTime() * 1_000)), TimeMicroVector::setNull, Arrays.asList(new Time(-128_000), new Time(104_000), new Time(-42_000))); - testSimpleType(new ArrowType.Time(TimeUnit.NANOSECOND, 64), Types.TIME, + testSimpleType( + new ArrowType.Time(TimeUnit.NANOSECOND, 64), + Types.TIME, (valueVectors, index, value) -> valueVectors.setSafe(index, (value.getTime() * 1_000_000)), TimeNanoVector::setNull, Arrays.asList(new Time(-128), new Time(104), new Time(-42))); @@ -286,57 +313,92 @@ void time64() throws SQLException { @Test void date32() throws SQLException { - testSimpleType(new ArrowType.Date(DateUnit.DAY), Types.DATE, - (valueVectors, index, value) -> valueVectors.setSafe(index, (int) (value.getTime() / MILLIS_PER_DAY)), + testSimpleType( + new ArrowType.Date(DateUnit.DAY), + Types.DATE, + (valueVectors, index, value) -> + valueVectors.setSafe(index, (int) (value.getTime() / MILLIS_PER_DAY)), DateDayVector::setNull, - Arrays.asList(new Date(-5 * MILLIS_PER_DAY), new Date(2 * MILLIS_PER_DAY), new Date(MILLIS_PER_DAY))); + Arrays.asList( + new Date(-5 * MILLIS_PER_DAY), new Date(2 * MILLIS_PER_DAY), new Date(MILLIS_PER_DAY))); } @Test void date64() throws SQLException { - testSimpleType(new ArrowType.Date(DateUnit.MILLISECOND), Types.DATE, + testSimpleType( + new ArrowType.Date(DateUnit.MILLISECOND), + Types.DATE, (valueVectors, index, value) -> valueVectors.setSafe(index, value.getTime()), DateMilliVector::setNull, - Arrays.asList(new Date(-5 * MILLIS_PER_DAY), new Date(2 * MILLIS_PER_DAY), new Date(MILLIS_PER_DAY))); + Arrays.asList( + new Date(-5 * MILLIS_PER_DAY), new Date(2 * MILLIS_PER_DAY), new Date(MILLIS_PER_DAY))); } @Test void timestamp() throws SQLException { - List values = Arrays.asList(new Timestamp(-128_000), new Timestamp(104_000), new Timestamp(-42_000)); - testSimpleType(new ArrowType.Timestamp(TimeUnit.SECOND, null), Types.TIMESTAMP, + List values = + Arrays.asList(new Timestamp(-128_000), new Timestamp(104_000), new Timestamp(-42_000)); + testSimpleType( + new ArrowType.Timestamp(TimeUnit.SECOND, null), + Types.TIMESTAMP, (valueVectors, index, value) -> valueVectors.setSafe(index, value.getTime() / 1_000), - TimeStampSecVector::setNull, values); - testSimpleType(new ArrowType.Timestamp(TimeUnit.MILLISECOND, null), Types.TIMESTAMP, + TimeStampSecVector::setNull, + values); + testSimpleType( + new ArrowType.Timestamp(TimeUnit.MILLISECOND, null), + Types.TIMESTAMP, (valueVectors, index, value) -> valueVectors.setSafe(index, value.getTime()), - TimeStampMilliVector::setNull, values); - testSimpleType(new ArrowType.Timestamp(TimeUnit.MICROSECOND, null), Types.TIMESTAMP, + TimeStampMilliVector::setNull, + values); + testSimpleType( + new ArrowType.Timestamp(TimeUnit.MICROSECOND, null), + Types.TIMESTAMP, (valueVectors, index, value) -> valueVectors.setSafe(index, value.getTime() * 1_000), - TimeStampMicroVector::setNull, values); - testSimpleType(new ArrowType.Timestamp(TimeUnit.NANOSECOND, null), Types.TIMESTAMP, + TimeStampMicroVector::setNull, + values); + testSimpleType( + new ArrowType.Timestamp(TimeUnit.NANOSECOND, null), + Types.TIMESTAMP, (valueVectors, index, value) -> valueVectors.setSafe(index, value.getTime() * 1_000_000), - TimeStampNanoVector::setNull, values); + TimeStampNanoVector::setNull, + values); } @Test void timestampTz() throws SQLException { - List values = Arrays.asList(new Timestamp(-128_000), new Timestamp(104_000), new Timestamp(-42_000)); - testSimpleType(new ArrowType.Timestamp(TimeUnit.SECOND, "UTC"), Types.TIMESTAMP_WITH_TIMEZONE, + List values = + Arrays.asList(new Timestamp(-128_000), new Timestamp(104_000), new Timestamp(-42_000)); + testSimpleType( + new ArrowType.Timestamp(TimeUnit.SECOND, "UTC"), + Types.TIMESTAMP_WITH_TIMEZONE, (valueVectors, index, value) -> valueVectors.setSafe(index, value.getTime() / 1_000), - TimeStampSecTZVector::setNull, values); - testSimpleType(new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC"), Types.TIMESTAMP_WITH_TIMEZONE, + TimeStampSecTZVector::setNull, + values); + testSimpleType( + new ArrowType.Timestamp(TimeUnit.MILLISECOND, "UTC"), + Types.TIMESTAMP_WITH_TIMEZONE, (valueVectors, index, value) -> valueVectors.setSafe(index, value.getTime()), - TimeStampMilliTZVector::setNull, values); - testSimpleType(new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC"), Types.TIMESTAMP_WITH_TIMEZONE, + TimeStampMilliTZVector::setNull, + values); + testSimpleType( + new ArrowType.Timestamp(TimeUnit.MICROSECOND, "UTC"), + Types.TIMESTAMP_WITH_TIMEZONE, (valueVectors, index, value) -> valueVectors.setSafe(index, value.getTime() * 1_000), - TimeStampMicroTZVector::setNull, values); - testSimpleType(new ArrowType.Timestamp(TimeUnit.NANOSECOND, "UTC"), Types.TIMESTAMP_WITH_TIMEZONE, + TimeStampMicroTZVector::setNull, + values); + testSimpleType( + new ArrowType.Timestamp(TimeUnit.NANOSECOND, "UTC"), + Types.TIMESTAMP_WITH_TIMEZONE, (valueVectors, index, value) -> valueVectors.setSafe(index, value.getTime() * 1_000_000), - TimeStampNanoTZVector::setNull, values); + TimeStampNanoTZVector::setNull, + values); } @Test void utf8() throws SQLException { - testSimpleType(ArrowType.Utf8.INSTANCE, Types.VARCHAR, + testSimpleType( + ArrowType.Utf8.INSTANCE, + Types.VARCHAR, (VarCharVector vector, Integer index, String value) -> vector.setSafe(index, value.getBytes(StandardCharsets.UTF_8)), BaseVariableWidthVector::setNull, @@ -345,7 +407,9 @@ void utf8() throws SQLException { @Test void largeUtf8() throws SQLException { - testSimpleType(ArrowType.LargeUtf8.INSTANCE, Types.LONGVARCHAR, + testSimpleType( + ArrowType.LargeUtf8.INSTANCE, + Types.LONGVARCHAR, (LargeVarCharVector vector, Integer index, String value) -> vector.setSafe(index, value.getBytes(StandardCharsets.UTF_8)), BaseLargeVariableWidthVector::setNull, @@ -354,155 +418,200 @@ void largeUtf8() throws SQLException { @Test void binary() throws SQLException { - testSimpleType(ArrowType.Binary.INSTANCE, Types.VARBINARY, - (VarBinaryVector vector, Integer index, byte[] value) -> - vector.setSafe(index, value), + testSimpleType( + ArrowType.Binary.INSTANCE, + Types.VARBINARY, + (VarBinaryVector vector, Integer index, byte[] value) -> vector.setSafe(index, value), BaseVariableWidthVector::setNull, Arrays.asList(new byte[0], new byte[] {2, -4}, new byte[] {0, -1, 127, -128})); } @Test void largeBinary() throws SQLException { - testSimpleType(ArrowType.LargeBinary.INSTANCE, Types.LONGVARBINARY, - (LargeVarBinaryVector vector, Integer index, byte[] value) -> - vector.setSafe(index, value), + testSimpleType( + ArrowType.LargeBinary.INSTANCE, + Types.LONGVARBINARY, + (LargeVarBinaryVector vector, Integer index, byte[] value) -> vector.setSafe(index, value), BaseLargeVariableWidthVector::setNull, Arrays.asList(new byte[0], new byte[] {2, -4}, new byte[] {0, -1, 127, -128})); } @Test void fixedSizeBinary() throws SQLException { - testSimpleType(new ArrowType.FixedSizeBinary(3), Types.BINARY, - FixedSizeBinaryVector::setSafe, FixedSizeBinaryVector::setNull, + testSimpleType( + new ArrowType.FixedSizeBinary(3), + Types.BINARY, + FixedSizeBinaryVector::setSafe, + FixedSizeBinaryVector::setNull, Arrays.asList(new byte[3], new byte[] {1, 2, -4}, new byte[] {-1, 127, -128})); } @Test void decimal128() throws SQLException { - testSimpleType(new ArrowType.Decimal(/*precision*/ 12, /*scale*/3, 128), Types.DECIMAL, - DecimalVector::setSafe, DecimalVector::setNull, - Arrays.asList(new BigDecimal("120.429"), new BigDecimal("-10590.123"), new BigDecimal("0.000"))); + testSimpleType( + new ArrowType.Decimal(/*precision*/ 12, /*scale*/ 3, 128), + Types.DECIMAL, + DecimalVector::setSafe, + DecimalVector::setNull, + Arrays.asList( + new BigDecimal("120.429"), new BigDecimal("-10590.123"), new BigDecimal("0.000"))); } @Test void decimal256() throws SQLException { - testSimpleType(new ArrowType.Decimal(/*precision*/ 12, /*scale*/3, 256), Types.DECIMAL, - Decimal256Vector::setSafe, Decimal256Vector::setNull, - Arrays.asList(new BigDecimal("120.429"), new BigDecimal("-10590.123"), new BigDecimal("0.000"))); + testSimpleType( + new ArrowType.Decimal(/*precision*/ 12, /*scale*/ 3, 256), + Types.DECIMAL, + Decimal256Vector::setSafe, + Decimal256Vector::setNull, + Arrays.asList( + new BigDecimal("120.429"), new BigDecimal("-10590.123"), new BigDecimal("0.000"))); } @Test void listOfDouble() throws SQLException { - TriConsumer setValue = (listVector, index, values) -> { - org.apache.arrow.vector.complex.impl.UnionListWriter writer = listVector.getWriter(); - writer.setPosition(index); - writer.startList(); - Arrays.stream(values).forEach(doubleValue -> writer.float8().writeFloat8(doubleValue)); - writer.endList(); - listVector.setLastSet(index); - }; - List values = Arrays.asList(new Double[]{0.0, Math.PI}, new Double[]{1.1, -352346.2, 2355.6}, - new Double[]{-1024.3}, new Double[]{}); - testListType(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE), setValue, ListVector::setNull, values); + TriConsumer setValue = + (listVector, index, values) -> { + org.apache.arrow.vector.complex.impl.UnionListWriter writer = listVector.getWriter(); + writer.setPosition(index); + writer.startList(); + Arrays.stream(values).forEach(doubleValue -> writer.float8().writeFloat8(doubleValue)); + writer.endList(); + listVector.setLastSet(index); + }; + List values = + Arrays.asList( + new Double[] {0.0, Math.PI}, new Double[] {1.1, -352346.2, 2355.6}, + new Double[] {-1024.3}, new Double[] {}); + testListType( + new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE), + setValue, + ListVector::setNull, + values); } @Test void listOfInt64() throws SQLException { - TriConsumer setValue = (listVector, index, values) -> { - org.apache.arrow.vector.complex.impl.UnionListWriter writer = listVector.getWriter(); - writer.setPosition(index); - writer.startList(); - Arrays.stream(values).forEach(longValue -> writer.bigInt().writeBigInt(longValue)); - writer.endList(); - listVector.setLastSet(index); - }; - List values = Arrays.asList(new Long[]{1L, 2L, 3L}, new Long[]{4L, 5L}, - new Long[]{512L, 1024L, 2048L, 4096L}, new Long[]{}); + TriConsumer setValue = + (listVector, index, values) -> { + org.apache.arrow.vector.complex.impl.UnionListWriter writer = listVector.getWriter(); + writer.setPosition(index); + writer.startList(); + Arrays.stream(values).forEach(longValue -> writer.bigInt().writeBigInt(longValue)); + writer.endList(); + listVector.setLastSet(index); + }; + List values = + Arrays.asList( + new Long[] {1L, 2L, 3L}, + new Long[] {4L, 5L}, + new Long[] {512L, 1024L, 2048L, 4096L}, + new Long[] {}); testListType((ArrowType) new ArrowType.Int(64, true), setValue, ListVector::setNull, values); } @Test void listOfInt32() throws SQLException { - TriConsumer setValue = (listVector, index, values) -> { - org.apache.arrow.vector.complex.impl.UnionListWriter writer = listVector.getWriter(); - writer.setPosition(index); - writer.startList(); - Arrays.stream(values).forEach(integerValue -> writer.integer().writeInt(integerValue)); - writer.endList(); - listVector.setLastSet(index); - }; - List values = Arrays.asList(new Integer[]{1, 2, 3}, new Integer[]{4, 5}, - new Integer[]{512, 1024, 2048, 4096}, new Integer[]{}); + TriConsumer setValue = + (listVector, index, values) -> { + org.apache.arrow.vector.complex.impl.UnionListWriter writer = listVector.getWriter(); + writer.setPosition(index); + writer.startList(); + Arrays.stream(values).forEach(integerValue -> writer.integer().writeInt(integerValue)); + writer.endList(); + listVector.setLastSet(index); + }; + List values = + Arrays.asList( + new Integer[] {1, 2, 3}, + new Integer[] {4, 5}, + new Integer[] {512, 1024, 2048, 4096}, + new Integer[] {}); testListType((ArrowType) new ArrowType.Int(32, true), setValue, ListVector::setNull, values); } @Test void listOfBoolean() throws SQLException { - TriConsumer setValue = (listVector, index, values) -> { - org.apache.arrow.vector.complex.impl.UnionListWriter writer = listVector.getWriter(); - writer.setPosition(index); - writer.startList(); - Arrays.stream(values).forEach(booleanValue -> writer.bit().writeBit(booleanValue ? 1 : 0)); - writer.endList(); - listVector.setLastSet(index); - }; - List values = Arrays.asList(new Boolean[]{true, false}, - new Boolean[]{false, false}, new Boolean[]{true, true, false, true}, new Boolean[]{}); + TriConsumer setValue = + (listVector, index, values) -> { + org.apache.arrow.vector.complex.impl.UnionListWriter writer = listVector.getWriter(); + writer.setPosition(index); + writer.startList(); + Arrays.stream(values) + .forEach(booleanValue -> writer.bit().writeBit(booleanValue ? 1 : 0)); + writer.endList(); + listVector.setLastSet(index); + }; + List values = + Arrays.asList( + new Boolean[] {true, false}, + new Boolean[] {false, false}, + new Boolean[] {true, true, false, true}, + new Boolean[] {}); testListType((ArrowType) new ArrowType.Bool(), setValue, ListVector::setNull, values); } @Test void listOfString() throws SQLException { - TriConsumer setValue = (listVector, index, values) -> { - org.apache.arrow.vector.complex.impl.UnionListWriter writer = listVector.getWriter(); - writer.setPosition(index); - writer.startList(); - Arrays.stream(values).forEach(stringValue -> { - if (stringValue != null) { - byte[] stringValueBytes = stringValue.getBytes(StandardCharsets.UTF_8); - try (ArrowBuf stringBuffer = allocator.buffer(stringValueBytes.length)) { - stringBuffer.writeBytes(stringValueBytes); - writer.varChar().writeVarChar(0, stringValueBytes.length, stringBuffer); - } - } else { - writer.varChar().writeNull(); - } - }); - writer.endList(); - listVector.setLastSet(index); - }; - List values = Arrays.asList(new String[]{"aaaa", "b1"}, - new String[]{"c", null, "d"}, new String[]{"e", "f", "g", "h"}, new String[]{}); + TriConsumer setValue = + (listVector, index, values) -> { + org.apache.arrow.vector.complex.impl.UnionListWriter writer = listVector.getWriter(); + writer.setPosition(index); + writer.startList(); + Arrays.stream(values) + .forEach( + stringValue -> { + if (stringValue != null) { + byte[] stringValueBytes = stringValue.getBytes(StandardCharsets.UTF_8); + try (ArrowBuf stringBuffer = allocator.buffer(stringValueBytes.length)) { + stringBuffer.writeBytes(stringValueBytes); + writer.varChar().writeVarChar(0, stringValueBytes.length, stringBuffer); + } + } else { + writer.varChar().writeNull(); + } + }); + writer.endList(); + listVector.setLastSet(index); + }; + List values = + Arrays.asList( + new String[] {"aaaa", "b1"}, + new String[] {"c", null, "d"}, + new String[] {"e", "f", "g", "h"}, + new String[] {}); testListType((ArrowType) new ArrowType.Utf8(), setValue, ListVector::setNull, values); } @Test void mapOfString() throws SQLException { - TriConsumer> setValue = (mapVector, index, values) -> { - org.apache.arrow.vector.complex.impl.UnionMapWriter mapWriter = mapVector.getWriter(); - mapWriter.setPosition(index); - mapWriter.startMap(); - values.entrySet().forEach(mapValue -> { - if (mapValue != null) { - byte[] keyBytes = mapValue.getKey().getBytes(StandardCharsets.UTF_8); - byte[] valueBytes = mapValue.getValue().getBytes(StandardCharsets.UTF_8); - try ( - ArrowBuf keyBuf = allocator.buffer(keyBytes.length); - ArrowBuf valueBuf = allocator.buffer(valueBytes.length); - ) { - mapWriter.startEntry(); - keyBuf.writeBytes(keyBytes); - valueBuf.writeBytes(valueBytes); - mapWriter.key().varChar().writeVarChar(0, keyBytes.length, keyBuf); - mapWriter.value().varChar().writeVarChar(0, valueBytes.length, valueBuf); - mapWriter.endEntry(); - } - } else { - mapWriter.writeNull(); - } - }); - mapWriter.endMap(); - }; + TriConsumer> setValue = + (mapVector, index, values) -> { + org.apache.arrow.vector.complex.impl.UnionMapWriter mapWriter = mapVector.getWriter(); + mapWriter.setPosition(index); + mapWriter.startMap(); + values + .entrySet() + .forEach( + mapValue -> { + if (mapValue != null) { + byte[] keyBytes = mapValue.getKey().getBytes(StandardCharsets.UTF_8); + byte[] valueBytes = mapValue.getValue().getBytes(StandardCharsets.UTF_8); + try (ArrowBuf keyBuf = allocator.buffer(keyBytes.length); + ArrowBuf valueBuf = allocator.buffer(valueBytes.length); ) { + mapWriter.startEntry(); + keyBuf.writeBytes(keyBytes); + valueBuf.writeBytes(valueBytes); + mapWriter.key().varChar().writeVarChar(0, keyBytes.length, keyBuf); + mapWriter.value().varChar().writeVarChar(0, valueBytes.length, valueBuf); + mapWriter.endEntry(); + } + } else { + mapWriter.writeNull(); + } + }); + mapWriter.endMap(); + }; JsonStringHashMap value1 = new JsonStringHashMap(); value1.put("a", "b"); @@ -514,28 +623,34 @@ void mapOfString() throws SQLException { JsonStringHashMap value3 = new JsonStringHashMap(); value3.put("y", "z"); value3.put("arrow", "cool"); - List> values = Arrays.asList(value1, value2, value3, Collections.emptyMap()); - testMapType(new ArrowType.Map(true), setValue, MapVector::setNull, values, new ArrowType.Utf8()); + List> values = + Arrays.asList(value1, value2, value3, Collections.emptyMap()); + testMapType( + new ArrowType.Map(true), setValue, MapVector::setNull, values, new ArrowType.Utf8()); } @Test void mapOfInteger() throws SQLException { - TriConsumer> setValue = (mapVector, index, values) -> { - org.apache.arrow.vector.complex.impl.UnionMapWriter mapWriter = mapVector.getWriter(); - mapWriter.setPosition(index); - mapWriter.startMap(); - values.entrySet().forEach(mapValue -> { - if (mapValue != null) { - mapWriter.startEntry(); - mapWriter.key().integer().writeInt(mapValue.getKey()); - mapWriter.value().integer().writeInt(mapValue.getValue()); - mapWriter.endEntry(); - } else { - mapWriter.writeNull(); - } - }); - mapWriter.endMap(); - }; + TriConsumer> setValue = + (mapVector, index, values) -> { + org.apache.arrow.vector.complex.impl.UnionMapWriter mapWriter = mapVector.getWriter(); + mapWriter.setPosition(index); + mapWriter.startMap(); + values + .entrySet() + .forEach( + mapValue -> { + if (mapValue != null) { + mapWriter.startEntry(); + mapWriter.key().integer().writeInt(mapValue.getKey()); + mapWriter.value().integer().writeInt(mapValue.getValue()); + mapWriter.endEntry(); + } else { + mapWriter.writeNull(); + } + }); + mapWriter.endMap(); + }; JsonStringHashMap value1 = new JsonStringHashMap(); value1.put(1, 2); @@ -547,8 +662,10 @@ void mapOfInteger() throws SQLException { JsonStringHashMap value3 = new JsonStringHashMap(); value3.put(Integer.MIN_VALUE, Integer.MAX_VALUE); value3.put(0, 4096); - List> values = Arrays.asList(value1, value2, value3, Collections.emptyMap()); - testMapType(new ArrowType.Map(true), setValue, MapVector::setNull, values, new ArrowType.Int(32, true)); + List> values = + Arrays.asList(value1, value2, value3, Collections.emptyMap()); + testMapType( + new ArrowType.Map(true), setValue, MapVector::setNull, values, new ArrowType.Int(32, true)); } @FunctionalInterface @@ -556,11 +673,16 @@ interface TriConsumer { void accept(T value1, U value2, V value3); } - void testSimpleType(ArrowType arrowType, int jdbcType, TriConsumer setValue, - BiConsumer setNull, List values) throws SQLException { + void testSimpleType( + ArrowType arrowType, + int jdbcType, + TriConsumer setValue, + BiConsumer setNull, + List values) + throws SQLException { Schema schema = new Schema(Collections.singletonList(Field.nullable("field", arrowType))); try (final MockPreparedStatement statement = new MockPreparedStatement(); - final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { + final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { final JdbcParameterBinder binder = JdbcParameterBinder.builder(statement, root).bindAll().build(); assertThat(binder.next()).isFalse(); @@ -610,7 +732,7 @@ void testSimpleType(ArrowType arrowType, int jdbcType // Non-nullable (since some types have a specialized binder) schema = new Schema(Collections.singletonList(Field.notNullable("field", arrowType))); try (final MockPreparedStatement statement = new MockPreparedStatement(); - final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { + final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { final JdbcParameterBinder binder = JdbcParameterBinder.builder(statement, root).bindAll().build(); assertThat(binder.next()).isFalse(); @@ -650,15 +772,23 @@ void testSimpleType(ArrowType arrowType, int jdbcType } } - void testListType(ArrowType arrowType, TriConsumer setValue, - BiConsumer setNull, List values) throws SQLException { + void testListType( + ArrowType arrowType, + TriConsumer setValue, + BiConsumer setNull, + List values) + throws SQLException { int jdbcType = Types.ARRAY; - Schema schema = new Schema(Collections.singletonList(new Field("field", FieldType.nullable( - new ArrowType.List()), Collections.singletonList( - new Field("element", FieldType.notNullable(arrowType), null) - )))); + Schema schema = + new Schema( + Collections.singletonList( + new Field( + "field", + FieldType.nullable(new ArrowType.List()), + Collections.singletonList( + new Field("element", FieldType.notNullable(arrowType), null))))); try (final MockPreparedStatement statement = new MockPreparedStatement(); - final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { + final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { final JdbcParameterBinder binder = JdbcParameterBinder.builder(statement, root).bindAll().build(); assertThat(binder.next()).isFalse(); @@ -706,12 +836,16 @@ void testListType(ArrowType arrowType, TriConsumer void testListType(ArrowType arrowType, TriConsumer void testMapType(ArrowType arrowType, TriConsumer setValue, - BiConsumer setNull, List values, - ArrowType elementType) throws SQLException { + void testMapType( + ArrowType arrowType, + TriConsumer setValue, + BiConsumer setNull, + List values, + ArrowType elementType) + throws SQLException { int jdbcType = Types.VARCHAR; FieldType keyType = new FieldType(false, elementType, null, null); FieldType mapType = new FieldType(false, ArrowType.Struct.INSTANCE, null, null); - Schema schema = new Schema(Collections.singletonList(new Field("field", FieldType.nullable(arrowType), - Collections.singletonList(new Field(MapVector.KEY_NAME, mapType, - Arrays.asList(new Field(MapVector.KEY_NAME, keyType, null), - new Field(MapVector.VALUE_NAME, keyType, null))))))); + Schema schema = + new Schema( + Collections.singletonList( + new Field( + "field", + FieldType.nullable(arrowType), + Collections.singletonList( + new Field( + MapVector.KEY_NAME, + mapType, + Arrays.asList( + new Field(MapVector.KEY_NAME, keyType, null), + new Field(MapVector.VALUE_NAME, keyType, null))))))); try (final MockPreparedStatement statement = new MockPreparedStatement(); - final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { + final VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) { final JdbcParameterBinder binder = JdbcParameterBinder.builder(statement, root).bindAll().build(); assertThat(binder.next()).isFalse(); @@ -810,18 +957,31 @@ void testMapType(ArrowType arrowType, TriConsumer metadata) { + private static Field field( + String name, boolean nullable, ArrowType type, Map metadata) { return new Field(name, new FieldType(nullable, type, null, metadata), Collections.emptyList()); } @@ -90,16 +91,26 @@ private static Map metadata(String... entries) { public void schemaComment() throws Exception { boolean includeMetadata = false; Schema schema = getSchemaWithCommentFromQuery(includeMetadata); - Schema expectedSchema = new Schema(Arrays.asList( - field("ID", false, Types.MinorType.BIGINT.getType(), - metadata("comment", "Record identifier")), - field("NAME", true, Types.MinorType.VARCHAR.getType(), - metadata("comment", "Name of record")), - field("COLUMN1", true, Types.MinorType.BIT.getType(), - metadata()), - field("COLUMNN", true, Types.MinorType.INT.getType(), - metadata("comment", "Informative description of columnN")) - ), metadata("comment", "This is super special table with valuable data")); + Schema expectedSchema = + new Schema( + Arrays.asList( + field( + "ID", + false, + Types.MinorType.BIGINT.getType(), + metadata("comment", "Record identifier")), + field( + "NAME", + true, + Types.MinorType.VARCHAR.getType(), + metadata("comment", "Name of record")), + field("COLUMN1", true, Types.MinorType.BIT.getType(), metadata()), + field( + "COLUMNN", + true, + Types.MinorType.INT.getType(), + metadata("comment", "Informative description of columnN"))), + metadata("comment", "This is super special table with valuable data")); assertThat(schema).isEqualTo(expectedSchema); } @@ -107,47 +118,60 @@ public void schemaComment() throws Exception { public void schemaCommentWithDatabaseMetadata() throws Exception { boolean includeMetadata = true; Schema schema = getSchemaWithCommentFromQuery(includeMetadata); - Schema expectedSchema = new Schema(Arrays.asList( - field("ID", false, Types.MinorType.BIGINT.getType(), - metadata( - "SQL_CATALOG_NAME", "JDBCTOARROWTEST?CHARACTERENCODING=UTF-8", - "SQL_SCHEMA_NAME", "PUBLIC", - "SQL_TABLE_NAME", "TABLE1", - "SQL_COLUMN_NAME", "ID", - "SQL_TYPE", "BIGINT", - "comment", "Record identifier" - )), - field("NAME", true, Types.MinorType.VARCHAR.getType(), - metadata( - "SQL_CATALOG_NAME", "JDBCTOARROWTEST?CHARACTERENCODING=UTF-8", - "SQL_SCHEMA_NAME", "PUBLIC", - "SQL_TABLE_NAME", "TABLE1", - "SQL_COLUMN_NAME", "NAME", - "SQL_TYPE", "CHARACTER VARYING", - "comment", "Name of record")), - field("COLUMN1", true, Types.MinorType.BIT.getType(), - metadata( - "SQL_CATALOG_NAME", "JDBCTOARROWTEST?CHARACTERENCODING=UTF-8", - "SQL_SCHEMA_NAME", "PUBLIC", - "SQL_TABLE_NAME", "TABLE1", - "SQL_COLUMN_NAME", "COLUMN1", - "SQL_TYPE", "BOOLEAN")), - field("COLUMNN", true, Types.MinorType.INT.getType(), - metadata( - "SQL_CATALOG_NAME", "JDBCTOARROWTEST?CHARACTERENCODING=UTF-8", - "SQL_SCHEMA_NAME", "PUBLIC", - "SQL_TABLE_NAME", "TABLE1", - "SQL_COLUMN_NAME", "COLUMNN", - "SQL_TYPE", "INTEGER", - "comment", "Informative description of columnN")) - ), metadata("comment", "This is super special table with valuable data")); + Schema expectedSchema = + new Schema( + Arrays.asList( + field( + "ID", + false, + Types.MinorType.BIGINT.getType(), + metadata( + "SQL_CATALOG_NAME", "JDBCTOARROWTEST?CHARACTERENCODING=UTF-8", + "SQL_SCHEMA_NAME", "PUBLIC", + "SQL_TABLE_NAME", "TABLE1", + "SQL_COLUMN_NAME", "ID", + "SQL_TYPE", "BIGINT", + "comment", "Record identifier")), + field( + "NAME", + true, + Types.MinorType.VARCHAR.getType(), + metadata( + "SQL_CATALOG_NAME", "JDBCTOARROWTEST?CHARACTERENCODING=UTF-8", + "SQL_SCHEMA_NAME", "PUBLIC", + "SQL_TABLE_NAME", "TABLE1", + "SQL_COLUMN_NAME", "NAME", + "SQL_TYPE", "CHARACTER VARYING", + "comment", "Name of record")), + field( + "COLUMN1", + true, + Types.MinorType.BIT.getType(), + metadata( + "SQL_CATALOG_NAME", "JDBCTOARROWTEST?CHARACTERENCODING=UTF-8", + "SQL_SCHEMA_NAME", "PUBLIC", + "SQL_TABLE_NAME", "TABLE1", + "SQL_COLUMN_NAME", "COLUMN1", + "SQL_TYPE", "BOOLEAN")), + field( + "COLUMNN", + true, + Types.MinorType.INT.getType(), + metadata( + "SQL_CATALOG_NAME", "JDBCTOARROWTEST?CHARACTERENCODING=UTF-8", + "SQL_SCHEMA_NAME", "PUBLIC", + "SQL_TABLE_NAME", "TABLE1", + "SQL_COLUMN_NAME", "COLUMNN", + "SQL_TYPE", "INTEGER", + "comment", "Informative description of columnN"))), + metadata("comment", "This is super special table with valuable data")); assertThat(schema).isEqualTo(expectedSchema); /* corresponding Apache Spark DDL after conversion: - ID BIGINT NOT NULL COMMENT 'Record identifier', - NAME STRING COMMENT 'Name of record', - COLUMN1 BOOLEAN, - COLUMNN INT COMMENT 'Informative description of columnN' - */ + ID BIGINT NOT NULL COMMENT 'Record identifier', + NAME STRING COMMENT 'Name of record', + COLUMN1 BOOLEAN, + COLUMNN INT COMMENT 'Informative description of columnN' + */ assertThat(schema).isEqualTo(expectedSchema); } @@ -156,19 +180,25 @@ private Schema getSchemaWithCommentFromQuery(boolean includeMetadata) throws SQL try (Statement statement = conn.createStatement()) { try (ResultSet resultSet = statement.executeQuery("select * from table1")) { ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); - Map> columnCommentByColumnIndex = getColumnComments(metaData, resultSetMetaData); + Map> columnCommentByColumnIndex = + getColumnComments(metaData, resultSetMetaData); String tableName = getTableNameFromResultSetMetaData(resultSetMetaData); String tableComment = getTableComment(metaData, tableName); - JdbcToArrowConfig config = new JdbcToArrowConfigBuilder() - .setAllocator(new RootAllocator()).setSchemaMetadata(Collections.singletonMap(COMMENT, tableComment)) - .setColumnMetadataByColumnIndex(columnCommentByColumnIndex).setIncludeMetadata(includeMetadata).build(); + JdbcToArrowConfig config = + new JdbcToArrowConfigBuilder() + .setAllocator(new RootAllocator()) + .setSchemaMetadata(Collections.singletonMap(COMMENT, tableComment)) + .setColumnMetadataByColumnIndex(columnCommentByColumnIndex) + .setIncludeMetadata(includeMetadata) + .build(); return JdbcToArrowUtils.jdbcToArrowSchema(resultSetMetaData, config); } } } - private String getTableNameFromResultSetMetaData(ResultSetMetaData resultSetMetaData) throws SQLException { + private String getTableNameFromResultSetMetaData(ResultSetMetaData resultSetMetaData) + throws SQLException { Set tablesFromQuery = new HashSet<>(); for (int idx = 1, columnCount = resultSetMetaData.getColumnCount(); idx <= columnCount; idx++) { String tableName = resultSetMetaData.getTableName(idx); @@ -182,11 +212,16 @@ private String getTableNameFromResultSetMetaData(ResultSetMetaData resultSetMeta throw new RuntimeException("Table metadata is absent or ambiguous"); } - private Map> getColumnComments(DatabaseMetaData metaData, - ResultSetMetaData resultSetMetaData) throws SQLException { + private Map> getColumnComments( + DatabaseMetaData metaData, ResultSetMetaData resultSetMetaData) throws SQLException { Map> columnCommentByColumnIndex = new HashMap<>(); - for (int columnIdx = 1, columnCount = resultSetMetaData.getColumnCount(); columnIdx <= columnCount; columnIdx++) { - String columnComment = getColumnComment(metaData, resultSetMetaData.getTableName(columnIdx), + for (int columnIdx = 1, columnCount = resultSetMetaData.getColumnCount(); + columnIdx <= columnCount; + columnIdx++) { + String columnComment = + getColumnComment( + metaData, + resultSetMetaData.getTableName(columnIdx), resultSetMetaData.getColumnName(columnIdx)); if (columnComment != null && !columnComment.isEmpty()) { columnCommentByColumnIndex.put(columnIdx, Collections.singletonMap(COMMENT, columnComment)); @@ -216,7 +251,8 @@ private String getTableComment(DatabaseMetaData metaData, String tableName) thro throw new RuntimeException("Table comment not found"); } - private String getColumnComment(DatabaseMetaData metaData, String tableName, String columnName) throws SQLException { + private String getColumnComment(DatabaseMetaData metaData, String tableName, String columnName) + throws SQLException { try (ResultSet tableMetadata = metaData.getColumns(null, null, tableName, columnName)) { if (tableMetadata.next()) { return tableMetadata.getString("REMARKS"); diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigTest.java index d4fb7c32997a7..85d6d89d036ff 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowConfigTest.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc; import static org.junit.Assert.assertEquals; @@ -28,7 +27,6 @@ import java.util.HashMap; import java.util.Locale; import java.util.TimeZone; - import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.junit.Test; @@ -36,7 +34,8 @@ public class JdbcToArrowConfigTest { private static final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE); - private static final Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT); + private static final Calendar calendar = + Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT); @Test(expected = NullPointerException.class) public void testConfigNullArguments() { @@ -116,13 +115,29 @@ public void testIncludeMetadata() { config = new JdbcToArrowConfigBuilder(allocator, calendar, true).build(); assertTrue(config.shouldIncludeMetadata()); - config = new JdbcToArrowConfig(allocator, calendar, /* include metadata */ true, - /* reuse vector schema root */ true, null, null, JdbcToArrowConfig.NO_LIMIT_BATCH_SIZE, null); + config = + new JdbcToArrowConfig( + allocator, + calendar, /* include metadata */ + true, + /* reuse vector schema root */ true, + null, + null, + JdbcToArrowConfig.NO_LIMIT_BATCH_SIZE, + null); assertTrue(config.shouldIncludeMetadata()); assertTrue(config.isReuseVectorSchemaRoot()); - config = new JdbcToArrowConfig(allocator, calendar, /* include metadata */ false, - /* reuse vector schema root */ false, null, null, JdbcToArrowConfig.NO_LIMIT_BATCH_SIZE, null); + config = + new JdbcToArrowConfig( + allocator, + calendar, /* include metadata */ + false, + /* reuse vector schema root */ false, + null, + null, + JdbcToArrowConfig.NO_LIMIT_BATCH_SIZE, + null); assertFalse(config.shouldIncludeMetadata()); assertFalse(config.isReuseVectorSchemaRoot()); } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java index 7dd881b3f7cec..375463d6fd5d4 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/JdbcToArrowTestHelper.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc; import static org.junit.Assert.assertArrayEquals; @@ -22,6 +21,9 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import java.math.BigDecimal; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; @@ -32,7 +34,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - import org.apache.arrow.vector.BaseValueVector; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.BitVector; @@ -58,12 +59,9 @@ import org.apache.arrow.vector.util.ObjectMapperFactory; import org.apache.arrow.vector.util.Text; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - /** - * This is a Helper class which has functionalities to read and assert the values from the given FieldVector object. + * This is a Helper class which has functionalities to read and assert the values from the given + * FieldVector object. */ public class JdbcToArrowTestHelper { @@ -79,7 +77,8 @@ public static void assertIntVectorValues(IntVector intVector, int rowCount, Inte } } - public static void assertBooleanVectorValues(BitVector bitVector, int rowCount, Boolean[] values) { + public static void assertBooleanVectorValues( + BitVector bitVector, int rowCount, Boolean[] values) { assertEquals(rowCount, bitVector.getValueCount()); for (int j = 0; j < bitVector.getValueCount(); j++) { @@ -103,7 +102,8 @@ public static void assertBitVectorValues(BitVector bitVector, int rowCount, Inte } } - public static void assertTinyIntVectorValues(TinyIntVector tinyIntVector, int rowCount, Integer[] values) { + public static void assertTinyIntVectorValues( + TinyIntVector tinyIntVector, int rowCount, Integer[] values) { assertEquals(rowCount, tinyIntVector.getValueCount()); for (int j = 0; j < tinyIntVector.getValueCount(); j++) { @@ -115,7 +115,8 @@ public static void assertTinyIntVectorValues(TinyIntVector tinyIntVector, int ro } } - public static void assertSmallIntVectorValues(SmallIntVector smallIntVector, int rowCount, Integer[] values) { + public static void assertSmallIntVectorValues( + SmallIntVector smallIntVector, int rowCount, Integer[] values) { assertEquals(rowCount, smallIntVector.getValueCount()); for (int j = 0; j < smallIntVector.getValueCount(); j++) { @@ -127,7 +128,8 @@ public static void assertSmallIntVectorValues(SmallIntVector smallIntVector, int } } - public static void assertBigIntVectorValues(BigIntVector bigIntVector, int rowCount, Long[] values) { + public static void assertBigIntVectorValues( + BigIntVector bigIntVector, int rowCount, Long[] values) { assertEquals(rowCount, bigIntVector.getValueCount()); for (int j = 0; j < bigIntVector.getValueCount(); j++) { @@ -139,7 +141,8 @@ public static void assertBigIntVectorValues(BigIntVector bigIntVector, int rowCo } } - public static void assertDecimalVectorValues(DecimalVector decimalVector, int rowCount, BigDecimal[] values) { + public static void assertDecimalVectorValues( + DecimalVector decimalVector, int rowCount, BigDecimal[] values) { assertEquals(rowCount, decimalVector.getValueCount()); for (int j = 0; j < decimalVector.getValueCount(); j++) { @@ -151,7 +154,8 @@ public static void assertDecimalVectorValues(DecimalVector decimalVector, int ro } } - public static void assertFloat8VectorValues(Float8Vector float8Vector, int rowCount, Double[] values) { + public static void assertFloat8VectorValues( + Float8Vector float8Vector, int rowCount, Double[] values) { assertEquals(rowCount, float8Vector.getValueCount()); for (int j = 0; j < float8Vector.getValueCount(); j++) { @@ -163,7 +167,8 @@ public static void assertFloat8VectorValues(Float8Vector float8Vector, int rowCo } } - public static void assertFloat4VectorValues(Float4Vector float4Vector, int rowCount, Float[] values) { + public static void assertFloat4VectorValues( + Float4Vector float4Vector, int rowCount, Float[] values) { assertEquals(rowCount, float4Vector.getValueCount()); for (int j = 0; j < float4Vector.getValueCount(); j++) { @@ -175,7 +180,8 @@ public static void assertFloat4VectorValues(Float4Vector float4Vector, int rowCo } } - public static void assertTimeVectorValues(TimeMilliVector timeMilliVector, int rowCount, Long[] values) { + public static void assertTimeVectorValues( + TimeMilliVector timeMilliVector, int rowCount, Long[] values) { assertEquals(rowCount, timeMilliVector.getValueCount()); for (int j = 0; j < timeMilliVector.getValueCount(); j++) { @@ -187,7 +193,8 @@ public static void assertTimeVectorValues(TimeMilliVector timeMilliVector, int r } } - public static void assertDateVectorValues(DateDayVector dateDayVector, int rowCount, Integer[] values) { + public static void assertDateVectorValues( + DateDayVector dateDayVector, int rowCount, Integer[] values) { assertEquals(rowCount, dateDayVector.getValueCount()); for (int j = 0; j < dateDayVector.getValueCount(); j++) { @@ -199,7 +206,8 @@ public static void assertDateVectorValues(DateDayVector dateDayVector, int rowCo } } - public static void assertTimeStampVectorValues(TimeStampVector timeStampVector, int rowCount, Long[] values) { + public static void assertTimeStampVectorValues( + TimeStampVector timeStampVector, int rowCount, Long[] values) { assertEquals(rowCount, timeStampVector.getValueCount()); for (int j = 0; j < timeStampVector.getValueCount(); j++) { @@ -211,7 +219,8 @@ public static void assertTimeStampVectorValues(TimeStampVector timeStampVector, } } - public static void assertVarBinaryVectorValues(VarBinaryVector varBinaryVector, int rowCount, byte[][] values) { + public static void assertVarBinaryVectorValues( + VarBinaryVector varBinaryVector, int rowCount, byte[][] values) { assertEquals(rowCount, varBinaryVector.getValueCount()); for (int j = 0; j < varBinaryVector.getValueCount(); j++) { @@ -223,7 +232,8 @@ public static void assertVarBinaryVectorValues(VarBinaryVector varBinaryVector, } } - public static void assertVarcharVectorValues(VarCharVector varCharVector, int rowCount, byte[][] values) { + public static void assertVarcharVectorValues( + VarCharVector varCharVector, int rowCount, byte[][] values) { assertEquals(rowCount, varCharVector.getValueCount()); for (int j = 0; j < varCharVector.getValueCount(); j++) { @@ -239,7 +249,8 @@ public static void assertNullVectorValues(NullVector vector, int rowCount) { assertEquals(rowCount, vector.getValueCount()); } - public static void assertListVectorValues(ListVector listVector, int rowCount, Integer[][] values) { + public static void assertListVectorValues( + ListVector listVector, int rowCount, Integer[][] values) { assertEquals(rowCount, listVector.getValueCount()); for (int j = 0; j < listVector.getValueCount(); j++) { @@ -252,7 +263,8 @@ public static void assertListVectorValues(ListVector listVector, int rowCount, I } } - public static void assertMapVectorValues(MapVector mapVector, int rowCount, Map[] values) { + public static void assertMapVectorValues( + MapVector mapVector, int rowCount, Map[] values) { assertEquals(rowCount, mapVector.getValueCount()); for (int j = 0; j < mapVector.getValueCount(); j++) { @@ -263,10 +275,17 @@ public static void assertMapVectorValues(MapVector mapVector, int rowCount, Map< (JsonStringArrayList>) mapVector.getObject(j); Map actualMap = null; if (actualSource != null && !actualSource.isEmpty()) { - actualMap = actualSource.stream().map(entry -> - new AbstractMap.SimpleEntry<>(entry.get("key").toString(), - entry.get("value") != null ? entry.get("value").toString() : null)) - .collect(HashMap::new, (collector, val) -> collector.put(val.getKey(), val.getValue()), HashMap::putAll); + actualMap = + actualSource.stream() + .map( + entry -> + new AbstractMap.SimpleEntry<>( + entry.get("key").toString(), + entry.get("value") != null ? entry.get("value").toString() : null)) + .collect( + HashMap::new, + (collector, val) -> collector.put(val.getKey(), val.getValue()), + HashMap::putAll); } assertEquals(values[j], actualMap); } @@ -310,8 +329,8 @@ public static void assertFieldMetadataIsEmpty(VectorSchemaRoot schema) { } } - public static void assertFieldMetadataMatchesResultSetMetadata(ResultSetMetaData rsmd, Schema schema) - throws SQLException { + public static void assertFieldMetadataMatchesResultSetMetadata( + ResultSetMetaData rsmd, Schema schema) throws SQLException { assertNotNull(schema); assertNotNull(schema.getFields()); assertNotNull(rsmd); @@ -400,12 +419,14 @@ public static byte[][] getCharArray(String[] values, String dataType) { byte[][] valueArr = new byte[dataArr.length][]; int i = 0; for (String data : dataArr) { - valueArr[i++] = "null".equals(data.trim()) ? null : data.trim().getBytes(StandardCharsets.UTF_8); + valueArr[i++] = + "null".equals(data.trim()) ? null : data.trim().getBytes(StandardCharsets.UTF_8); } return valueArr; } - public static byte[][] getCharArrayWithCharSet(String[] values, String dataType, Charset charSet) { + public static byte[][] getCharArrayWithCharSet( + String[] values, String dataType, Charset charSet) { String[] dataArr = getValues(values, dataType); byte[][] valueArr = new byte[dataArr.length][]; int i = 0; @@ -420,7 +441,8 @@ public static byte[][] getBinaryValues(String[] values, String dataType) { byte[][] valueArr = new byte[dataArr.length][]; int i = 0; for (String data : dataArr) { - valueArr[i++] = "null".equals(data.trim()) ? null : data.trim().getBytes(StandardCharsets.UTF_8); + valueArr[i++] = + "null".equals(data.trim()) ? null : data.trim().getBytes(StandardCharsets.UTF_8); } return valueArr; } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/MockPreparedStatement.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/MockPreparedStatement.java index 4478cdfbee6f7..8dfc684e22f24 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/MockPreparedStatement.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/MockPreparedStatement.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc; import java.io.InputStream; @@ -231,8 +230,7 @@ public void setDate(int parameterIndex, Date x, Calendar cal) throws SQLExceptio } @Override - public void setTime(int parameterIndex, Time x, Calendar cal) throws SQLException { - } + public void setTime(int parameterIndex, Time x, Calendar cal) throws SQLException {} @Override public void setTimestamp(int parameterIndex, Timestamp x, Calendar cal) throws SQLException { @@ -242,8 +240,7 @@ public void setTimestamp(int parameterIndex, Timestamp x, Calendar cal) throws S } @Override - public void setNull(int parameterIndex, int sqlType, String typeName) throws SQLException { - } + public void setNull(int parameterIndex, int sqlType, String typeName) throws SQLException {} @Override public void setURL(int parameterIndex, URL x) throws SQLException { @@ -261,80 +258,62 @@ public void setRowId(int parameterIndex, RowId x) throws SQLException { } @Override - public void setNString(int parameterIndex, String value) throws SQLException { - } + public void setNString(int parameterIndex, String value) throws SQLException {} @Override public void setNCharacterStream(int parameterIndex, Reader value, long length) - throws SQLException { - } + throws SQLException {} @Override - public void setNClob(int parameterIndex, NClob value) throws SQLException { - } + public void setNClob(int parameterIndex, NClob value) throws SQLException {} @Override - public void setClob(int parameterIndex, Reader reader, long length) throws SQLException { - } + public void setClob(int parameterIndex, Reader reader, long length) throws SQLException {} @Override public void setBlob(int parameterIndex, InputStream inputStream, long length) - throws SQLException { - } + throws SQLException {} @Override - public void setNClob(int parameterIndex, Reader reader, long length) throws SQLException { - } + public void setNClob(int parameterIndex, Reader reader, long length) throws SQLException {} @Override - public void setSQLXML(int parameterIndex, SQLXML xmlObject) throws SQLException { - } + public void setSQLXML(int parameterIndex, SQLXML xmlObject) throws SQLException {} @Override public void setObject(int parameterIndex, Object x, int targetSqlType, int scaleOrLength) - throws SQLException { - } + throws SQLException {} @Override - public void setAsciiStream(int parameterIndex, InputStream x, long length) throws SQLException { - } + public void setAsciiStream(int parameterIndex, InputStream x, long length) throws SQLException {} @Override - public void setBinaryStream(int parameterIndex, InputStream x, long length) throws SQLException { - } + public void setBinaryStream(int parameterIndex, InputStream x, long length) throws SQLException {} @Override public void setCharacterStream(int parameterIndex, Reader reader, long length) - throws SQLException { - } + throws SQLException {} @Override - public void setAsciiStream(int parameterIndex, InputStream x) throws SQLException { - } + public void setAsciiStream(int parameterIndex, InputStream x) throws SQLException {} @Override - public void setBinaryStream(int parameterIndex, InputStream x) throws SQLException { - } + public void setBinaryStream(int parameterIndex, InputStream x) throws SQLException {} @Override - public void setCharacterStream(int parameterIndex, Reader reader) throws SQLException { - } + public void setCharacterStream(int parameterIndex, Reader reader) throws SQLException {} @Override - public void setNCharacterStream(int parameterIndex, Reader value) throws SQLException { - } + public void setNCharacterStream(int parameterIndex, Reader value) throws SQLException {} @Override - public void setClob(int parameterIndex, Reader reader) throws SQLException { - } + public void setClob(int parameterIndex, Reader reader) throws SQLException {} @Override - public void setBlob(int parameterIndex, InputStream inputStream) throws SQLException { - } + public void setBlob(int parameterIndex, InputStream inputStream) throws SQLException {} @Override - public void setNClob(int parameterIndex, Reader reader) throws SQLException { - } + public void setNClob(int parameterIndex, Reader reader) throws SQLException {} @Override public ResultSet executeQuery(String sql) throws SQLException { @@ -347,8 +326,7 @@ public int executeUpdate(String sql) throws SQLException { } @Override - public void close() throws SQLException { - } + public void close() throws SQLException {} @Override public int getMaxFieldSize() throws SQLException { diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ResultSetUtility.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ResultSetUtility.java index b05a59a9a04d8..5f5f6dcb98d43 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ResultSetUtility.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ResultSetUtility.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc; import java.io.InputStream; @@ -48,13 +47,11 @@ public class ResultSetUtility { public static ResultSet generateEmptyResultSet() throws SQLException { MockDataElement element = new MockDataElement("string_example"); MockResultSetMetaData.MockColumnMetaData columnMetaData = - MockResultSetMetaData.MockColumnMetaData.fromDataElement(element, 1); + MockResultSetMetaData.MockColumnMetaData.fromDataElement(element, 1); ArrayList cols = new ArrayList<>(); cols.add(columnMetaData); ResultSetMetaData metadata = new MockResultSetMetaData(cols); - return MockResultSet.builder() - .setMetaData(metadata) - .build(); + return MockResultSet.builder().setMetaData(metadata).build(); } public static MockResultSet generateBasicResultSet(int rows) throws SQLException { @@ -319,15 +316,19 @@ public String getColumnTypeName(int column) throws SQLException { } public static MockResultSetMetaData fromRows(List rows) throws SQLException { - // Note: This attempts to dynamically construct ResultSetMetaData from the first row in a given result set. - // If there are now rows, or the result set contains no columns, this cannot be dynamically generated and + // Note: This attempts to dynamically construct ResultSetMetaData from the first row in a + // given result set. + // If there are now rows, or the result set contains no columns, this cannot be dynamically + // generated and // an exception will be thrown. if (rows.size() == 0) { - throw new SQLException("Unable to dynamically generate ResultSetMetaData because row count is zero!"); + throw new SQLException( + "Unable to dynamically generate ResultSetMetaData because row count is zero!"); } MockRow firstRow = rows.get(0); if (firstRow.dataElements.size() == 0) { - throw new SQLException("Unable to dynamically generate ResultSetMetaData because column count is zero!"); + throw new SQLException( + "Unable to dynamically generate ResultSetMetaData because column count is zero!"); } ArrayList columns = new ArrayList<>(); for (int i = 0; i < firstRow.dataElements.size(); i++) { @@ -346,9 +347,7 @@ public static class MockColumnMetaData { private String typeName; private int displaySize; - - private MockColumnMetaData() { - } + private MockColumnMetaData() {} private String getLabel() { return label; @@ -382,16 +381,17 @@ private int getDisplaySize() { return displaySize; } - public static MockColumnMetaData fromDataElement(MockDataElement element, int i) throws SQLException { + public static MockColumnMetaData fromDataElement(MockDataElement element, int i) + throws SQLException { return MockColumnMetaData.builder() - .sqlType(element.getSqlType()) - .precision(element.getPrecision()) - .scale(element.getScale()) - .nullable(element.isNullable()) - .setTypeName("TYPE") - .setDisplaySize(420) - .label("col_" + i) - .build(); + .sqlType(element.getSqlType()) + .precision(element.getPrecision()) + .scale(element.getScale()) + .nullable(element.isNullable()) + .setTypeName("TYPE") + .setDisplaySize(420) + .label("col_" + i) + .build(); } public static Builder builder() { @@ -440,9 +440,7 @@ public MockColumnMetaData build() { return this.columnMetaData; } } - } - } public static class MockRow { @@ -635,7 +633,6 @@ public short getShort() throws SQLException { } } - public static class ThrowingResultSet implements ResultSet { @Override @@ -1139,17 +1136,20 @@ public void updateTimestamp(String columnLabel, Timestamp x) throws SQLException } @Override - public void updateAsciiStream(String columnLabel, InputStream x, int length) throws SQLException { + public void updateAsciiStream(String columnLabel, InputStream x, int length) + throws SQLException { throw getExceptionToThrow(); } @Override - public void updateBinaryStream(String columnLabel, InputStream x, int length) throws SQLException { + public void updateBinaryStream(String columnLabel, InputStream x, int length) + throws SQLException { throw getExceptionToThrow(); } @Override - public void updateCharacterStream(String columnLabel, Reader reader, int length) throws SQLException { + public void updateCharacterStream(String columnLabel, Reader reader, int length) + throws SQLException { throw getExceptionToThrow(); } @@ -1439,7 +1439,8 @@ public void updateNCharacterStream(int columnIndex, Reader x, long length) throw } @Override - public void updateNCharacterStream(String columnLabel, Reader reader, long length) throws SQLException { + public void updateNCharacterStream(String columnLabel, Reader reader, long length) + throws SQLException { throw getExceptionToThrow(); } @@ -1449,7 +1450,8 @@ public void updateAsciiStream(int columnIndex, InputStream x, long length) throw } @Override - public void updateBinaryStream(int columnIndex, InputStream x, long length) throws SQLException { + public void updateBinaryStream(int columnIndex, InputStream x, long length) + throws SQLException { throw getExceptionToThrow(); } @@ -1459,27 +1461,32 @@ public void updateCharacterStream(int columnIndex, Reader x, long length) throws } @Override - public void updateAsciiStream(String columnLabel, InputStream x, long length) throws SQLException { + public void updateAsciiStream(String columnLabel, InputStream x, long length) + throws SQLException { throw getExceptionToThrow(); } @Override - public void updateBinaryStream(String columnLabel, InputStream x, long length) throws SQLException { + public void updateBinaryStream(String columnLabel, InputStream x, long length) + throws SQLException { throw getExceptionToThrow(); } @Override - public void updateCharacterStream(String columnLabel, Reader reader, long length) throws SQLException { + public void updateCharacterStream(String columnLabel, Reader reader, long length) + throws SQLException { throw getExceptionToThrow(); } @Override - public void updateBlob(int columnIndex, InputStream inputStream, long length) throws SQLException { + public void updateBlob(int columnIndex, InputStream inputStream, long length) + throws SQLException { throw getExceptionToThrow(); } @Override - public void updateBlob(String columnLabel, InputStream inputStream, long length) throws SQLException { + public void updateBlob(String columnLabel, InputStream inputStream, long length) + throws SQLException { throw getExceptionToThrow(); } @@ -1584,13 +1591,14 @@ public T getObject(String columnLabel, Class type) throws SQLException { } @Override - public void updateObject(int columnIndex, Object x, SQLType targetSqlType, int scaleOrLength) throws SQLException { + public void updateObject(int columnIndex, Object x, SQLType targetSqlType, int scaleOrLength) + throws SQLException { throw getExceptionToThrow(); } @Override public void updateObject(String columnLabel, Object x, SQLType targetSqlType, int scaleOrLength) - throws SQLException { + throws SQLException { throw getExceptionToThrow(); } @@ -1600,7 +1608,8 @@ public void updateObject(int columnIndex, Object x, SQLType targetSqlType) throw } @Override - public void updateObject(String columnLabel, Object x, SQLType targetSqlType) throws SQLException { + public void updateObject(String columnLabel, Object x, SQLType targetSqlType) + throws SQLException { throw getExceptionToThrow(); } @@ -1623,7 +1632,6 @@ private static SQLException getExceptionToThrow(String message) { return new SQLException(message); } - public static class ThrowingResultSetMetaData implements ResultSetMetaData { @Override public int getColumnCount() throws SQLException { diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ResultSetUtilityTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ResultSetUtilityTest.java index 2424ed625248d..8af2c06f4de54 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ResultSetUtilityTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/ResultSetUtilityTest.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc; import static org.junit.Assert.assertEquals; @@ -26,7 +25,6 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Types; - import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.VectorSchemaRoot; @@ -36,20 +34,24 @@ public class ResultSetUtilityTest { @Test public void testZeroRowResultSet() throws Exception { - for (boolean reuseVectorSchemaRoot : new boolean[]{false, true}) { + for (boolean reuseVectorSchemaRoot : new boolean[] {false, true}) { try (BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE)) { ResultSet rs = ResultSetUtility.generateEmptyResultSet(); - JdbcToArrowConfig config = new JdbcToArrowConfigBuilder( - allocator, JdbcToArrowUtils.getUtcCalendar(), /* include metadata */ false) - .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) - .build(); + JdbcToArrowConfig config = + new JdbcToArrowConfigBuilder( + allocator, JdbcToArrowUtils.getUtcCalendar(), /* include metadata */ false) + .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) + .build(); ArrowVectorIterator iter = JdbcToArrow.sqlToArrowVectorIterator(rs, config); assertTrue("Iterator on zero row ResultSet should haveNext() before use", iter.hasNext()); VectorSchemaRoot root = iter.next(); assertNotNull("VectorSchemaRoot from first next() result should never be null", root); - assertEquals("VectorSchemaRoot from empty ResultSet should have zero rows", 0, root.getRowCount()); - assertFalse("hasNext() should return false on empty ResultSets after initial next() call", iter.hasNext()); + assertEquals( + "VectorSchemaRoot from empty ResultSet should have zero rows", 0, root.getRowCount()); + assertFalse( + "hasNext() should return false on empty ResultSets after initial next() call", + iter.hasNext()); } } } @@ -99,7 +101,8 @@ public void testBasicResultSet() throws Exception { @Test public void testMockDataTypes() throws SQLException { - ResultSetUtility.MockDataElement element = new ResultSetUtility.MockDataElement(1L, Types.NUMERIC); + ResultSetUtility.MockDataElement element = + new ResultSetUtility.MockDataElement(1L, Types.NUMERIC); assertEquals(1L, element.getLong()); assertEquals(1, element.getInt()); assertEquals("1", element.getString()); diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java index 50c4fe6db2a14..7fa8188a99158 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/Table.java @@ -14,17 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.math.BigDecimal; import java.nio.charset.StandardCharsets; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; - -/** - * POJO to handle the YAML data from the test YAML file. - */ +/** POJO to handle the YAML data from the test YAML file. */ @JsonIgnoreProperties(ignoreUnknown = true) public class Table { private String name; @@ -39,8 +35,7 @@ public class Table { private String[] vectors; private int rowCount; - public Table() { - } + public Table() {} public String getName() { return name; diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/UnreliableMetaDataTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/UnreliableMetaDataTest.java index 053604073fd66..93ba028e39629 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/UnreliableMetaDataTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/UnreliableMetaDataTest.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc; import static org.junit.Assert.assertEquals; @@ -34,7 +33,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; - import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.IntVector; @@ -48,9 +46,7 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -/** - * Test options for dealing with unreliable ResultSetMetaData from JDBC drivers. - */ +/** Test options for dealing with unreliable ResultSetMetaData from JDBC drivers. */ @RunWith(Parameterized.class) public class UnreliableMetaDataTest { private final boolean reuseVectorSchemaRoot; @@ -72,7 +68,7 @@ public void afterEach() { @Parameterized.Parameters(name = "reuseVectorSchemaRoot = {0}") public static Collection getTestData() { - return Arrays.asList(new Object[][] { {false}, {true} }); + return Arrays.asList(new Object[][] {{false}, {true}}); } @Test @@ -91,13 +87,15 @@ public void testUnreliableMetaDataPrecisionAndScale() throws Exception { // reset the ResultSet: rs.beforeFirst(); - JdbcToArrowConfig config = new JdbcToArrowConfigBuilder( - allocator, JdbcToArrowUtils.getUtcCalendar(), /* include metadata */ false) - .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) - .build(); + JdbcToArrowConfig config = + new JdbcToArrowConfigBuilder( + allocator, JdbcToArrowUtils.getUtcCalendar(), /* include metadata */ false) + .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) + .build(); try (ArrowVectorIterator iter = JdbcToArrow.sqlToArrowVectorIterator(rs, config)) { assertTrue(iter.hasNext()); - assertThrows(RuntimeException.class, iter::next, "Expected to fail due to mismatched metadata!"); + assertThrows( + RuntimeException.class, iter::next, "Expected to fail due to mismatched metadata!"); } // reset the ResultSet: @@ -105,11 +103,12 @@ public void testUnreliableMetaDataPrecisionAndScale() throws Exception { JdbcFieldInfo explicitMappingField = new JdbcFieldInfo(Types.DECIMAL, 18, 2); Map explicitMapping = new HashMap<>(); explicitMapping.put(1, explicitMappingField); - config = new JdbcToArrowConfigBuilder( - allocator, JdbcToArrowUtils.getUtcCalendar(), /* include metadata */ false) - .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) - .setExplicitTypesByColumnIndex(explicitMapping) - .build(); + config = + new JdbcToArrowConfigBuilder( + allocator, JdbcToArrowUtils.getUtcCalendar(), /* include metadata */ false) + .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) + .setExplicitTypesByColumnIndex(explicitMapping) + .build(); try (ArrowVectorIterator iter = JdbcToArrow.sqlToArrowVectorIterator(rs, config)) { while (iter.hasNext()) { @@ -133,7 +132,8 @@ public void testInconsistentPrecisionAndScale() throws Exception { assertEquals("Value precision should be 18", 18, bd1.precision()); rs.next(); BigDecimal bd2 = rs.getBigDecimal(1); - assertEquals("Value should be 1000000000300.0000001", new BigDecimal("1000000000300.0000001"), bd2); + assertEquals( + "Value should be 1000000000300.0000001", new BigDecimal("1000000000300.0000001"), bd2); assertEquals("Value scale should be 7", 7, bd2.scale()); assertEquals("Value precision should be 20", 20, bd2.precision()); rs.beforeFirst(); @@ -141,23 +141,27 @@ public void testInconsistentPrecisionAndScale() throws Exception { Map explicitMapping = new HashMap<>(); explicitMapping.put(1, explicitMappingField); - JdbcToArrowConfig config = new JdbcToArrowConfigBuilder( - allocator, JdbcToArrowUtils.getUtcCalendar(), /* include metadata */ false) - .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) - .setExplicitTypesByColumnIndex(explicitMapping) - .build(); + JdbcToArrowConfig config = + new JdbcToArrowConfigBuilder( + allocator, JdbcToArrowUtils.getUtcCalendar(), /* include metadata */ false) + .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) + .setExplicitTypesByColumnIndex(explicitMapping) + .build(); try (ArrowVectorIterator iter = JdbcToArrow.sqlToArrowVectorIterator(rs, config)) { assertTrue(iter.hasNext()); - assertThrows(RuntimeException.class, iter::next, + assertThrows( + RuntimeException.class, + iter::next, "This is expected to fail due to inconsistent BigDecimal scales, while strict matching is enabled."); } // Reuse same ResultSet, with RoundingMode.UNNECESSARY set to coerce BigDecimal scale as needed: - config = new JdbcToArrowConfigBuilder( - allocator, JdbcToArrowUtils.getUtcCalendar(), /* include metadata */ false) - .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) - .setExplicitTypesByColumnIndex(explicitMapping) - .setBigDecimalRoundingMode(RoundingMode.UNNECESSARY) - .build(); + config = + new JdbcToArrowConfigBuilder( + allocator, JdbcToArrowUtils.getUtcCalendar(), /* include metadata */ false) + .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) + .setExplicitTypesByColumnIndex(explicitMapping) + .setBigDecimalRoundingMode(RoundingMode.UNNECESSARY) + .build(); try (ArrowVectorIterator iter = JdbcToArrow.sqlToArrowVectorIterator(rs, config)) { while (iter.hasNext()) { VectorSchemaRoot root = iter.next(); @@ -174,23 +178,29 @@ public void testIncorrectNullability() throws Exception { .sqlType(Types.INTEGER) .nullable(ResultSetMetaData.columnNoNulls) .build(); - ResultSetMetaData metadata = new ResultSetUtility.MockResultSetMetaData(Collections.singletonList(columnMetaData)); - final ResultSetUtility.MockResultSet.Builder resultSetBuilder = ResultSetUtility.MockResultSet.builder() - .setMetaData(metadata) - .addDataElement(new ResultSetUtility.MockDataElement(1024, Types.INTEGER)) - .finishRow() - .addDataElement(new ResultSetUtility.MockDataElement(null, Types.INTEGER)) - .finishRow(); - final Schema notNullSchema = new Schema( - Collections.singletonList(Field.notNullable(/*name=*/null, new ArrowType.Int(32, true)))); - final Schema nullSchema = new Schema( - Collections.singletonList(Field.nullable(/*name=*/null, new ArrowType.Int(32, true)))); + ResultSetMetaData metadata = + new ResultSetUtility.MockResultSetMetaData(Collections.singletonList(columnMetaData)); + final ResultSetUtility.MockResultSet.Builder resultSetBuilder = + ResultSetUtility.MockResultSet.builder() + .setMetaData(metadata) + .addDataElement(new ResultSetUtility.MockDataElement(1024, Types.INTEGER)) + .finishRow() + .addDataElement(new ResultSetUtility.MockDataElement(null, Types.INTEGER)) + .finishRow(); + final Schema notNullSchema = + new Schema( + Collections.singletonList( + Field.notNullable(/*name=*/ null, new ArrowType.Int(32, true)))); + final Schema nullSchema = + new Schema( + Collections.singletonList(Field.nullable(/*name=*/ null, new ArrowType.Int(32, true)))); try (final ResultSet rs = resultSetBuilder.build()) { - JdbcToArrowConfig config = new JdbcToArrowConfigBuilder( - allocator, JdbcToArrowUtils.getUtcCalendar(), /* include metadata */ false) - .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) - .build(); + JdbcToArrowConfig config = + new JdbcToArrowConfigBuilder( + allocator, JdbcToArrowUtils.getUtcCalendar(), /* include metadata */ false) + .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) + .build(); try (ArrowVectorIterator iter = JdbcToArrow.sqlToArrowVectorIterator(rs, config)) { assertTrue(iter.hasNext()); final VectorSchemaRoot root = iter.next(); @@ -208,14 +218,16 @@ public void testIncorrectNullability() throws Exception { // Override the nullability to get the correct result final Map typeMapping = new HashMap<>(); - JdbcFieldInfo realFieldInfo = new JdbcFieldInfo( - Types.INTEGER, ResultSetMetaData.columnNullable, /*precision*/0, /*scale*/0); + JdbcFieldInfo realFieldInfo = + new JdbcFieldInfo( + Types.INTEGER, ResultSetMetaData.columnNullable, /*precision*/ 0, /*scale*/ 0); typeMapping.put(1, realFieldInfo); - config = new JdbcToArrowConfigBuilder( - allocator, JdbcToArrowUtils.getUtcCalendar(), /* include metadata */ false) - .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) - .setExplicitTypesByColumnIndex(typeMapping) - .build(); + config = + new JdbcToArrowConfigBuilder( + allocator, JdbcToArrowUtils.getUtcCalendar(), /* include metadata */ false) + .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) + .setExplicitTypesByColumnIndex(typeMapping) + .build(); try (ArrowVectorIterator iter = JdbcToArrow.sqlToArrowVectorIterator(rs, config)) { assertTrue(iter.hasNext()); final VectorSchemaRoot root = iter.next(); @@ -231,14 +243,16 @@ public void testIncorrectNullability() throws Exception { rs.beforeFirst(); // columnNullableUnknown won't override the metadata - realFieldInfo = new JdbcFieldInfo( - Types.INTEGER, ResultSetMetaData.columnNullableUnknown, /*precision*/0, /*scale*/0); + realFieldInfo = + new JdbcFieldInfo( + Types.INTEGER, ResultSetMetaData.columnNullableUnknown, /*precision*/ 0, /*scale*/ 0); typeMapping.put(1, realFieldInfo); - config = new JdbcToArrowConfigBuilder( - allocator, JdbcToArrowUtils.getUtcCalendar(), /* include metadata */ false) - .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) - .setExplicitTypesByColumnIndex(typeMapping) - .build(); + config = + new JdbcToArrowConfigBuilder( + allocator, JdbcToArrowUtils.getUtcCalendar(), /* include metadata */ false) + .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) + .setExplicitTypesByColumnIndex(typeMapping) + .build(); try (ArrowVectorIterator iter = JdbcToArrow.sqlToArrowVectorIterator(rs, config)) { assertTrue(iter.hasNext()); final VectorSchemaRoot root = iter.next(); @@ -266,8 +280,8 @@ private ResultSet buildIncorrectPrecisionAndScaleMetaDataResultSet() throws SQLE return ResultSetUtility.MockResultSet.builder() .setMetaData(metadata) .addDataElement( - new ResultSetUtility.MockDataElement(new BigDecimal("1000000000000000.01"), Types.DECIMAL) - ) + new ResultSetUtility.MockDataElement( + new BigDecimal("1000000000000000.01"), Types.DECIMAL)) .finishRow() .build(); } @@ -285,12 +299,12 @@ private ResultSet buildVaryingPrecisionAndScaleResultSet() throws SQLException { return ResultSetUtility.MockResultSet.builder() .setMetaData(metadata) .addDataElement( - new ResultSetUtility.MockDataElement(new BigDecimal("1000000000000000.01"), Types.DECIMAL) - ) + new ResultSetUtility.MockDataElement( + new BigDecimal("1000000000000000.01"), Types.DECIMAL)) .finishRow() .addDataElement( - new ResultSetUtility.MockDataElement(new BigDecimal("1000000000300.0000001"), Types.DECIMAL) - ) + new ResultSetUtility.MockDataElement( + new BigDecimal("1000000000300.0000001"), Types.DECIMAL)) .finishRow() .build(); } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/consumer/AbstractConsumerTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/consumer/AbstractConsumerTest.java index 96bac42214cef..6a25c58fbde7e 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/consumer/AbstractConsumerTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/consumer/AbstractConsumerTest.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.consumer; import org.apache.arrow.memory.BufferAllocator; @@ -35,5 +34,4 @@ public void setUp() { public void tearDown() { allocator.close(); } - } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumerTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumerTest.java index a368023d49005..255770ecdbf6d 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumerTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/consumer/BinaryConsumerTest.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.consumer; import static org.junit.Assert.assertArrayEquals; @@ -23,7 +22,6 @@ import java.io.ByteArrayInputStream; import java.io.IOException; - import org.apache.arrow.vector.BaseValueVector; import org.apache.arrow.vector.VarBinaryVector; import org.junit.Test; @@ -37,7 +35,8 @@ interface InputStreamConsumer { void consume(BinaryConsumer consumer) throws IOException; } - protected void assertConsume(boolean nullable, InputStreamConsumer dataConsumer, byte[][] expect) throws IOException { + protected void assertConsume(boolean nullable, InputStreamConsumer dataConsumer, byte[][] expect) + throws IOException { try (final VarBinaryVector vector = new VarBinaryVector("binary", allocator)) { BinaryConsumer consumer = BinaryConsumer.createConsumer(vector, 0, nullable); dataConsumer.consume(consumer); @@ -61,51 +60,59 @@ private byte[] createBytes(int length) { return bytes; } - public void testConsumeInputStream(byte[][] values, boolean nullable) throws IOException { - assertConsume(nullable, binaryConsumer -> { - for (byte[] value : values) { - binaryConsumer.consume(new ByteArrayInputStream(value)); - binaryConsumer.moveWriterPosition(); - } - }, values); + assertConsume( + nullable, + binaryConsumer -> { + for (byte[] value : values) { + binaryConsumer.consume(new ByteArrayInputStream(value)); + binaryConsumer.moveWriterPosition(); + } + }, + values); } @Test public void testConsumeInputStream() throws IOException { - testConsumeInputStream(new byte[][]{ - createBytes(DEFAULT_RECORD_BYTE_COUNT) - }, false); - - testConsumeInputStream(new byte[][]{ - createBytes(DEFAULT_RECORD_BYTE_COUNT), - createBytes(DEFAULT_RECORD_BYTE_COUNT) - }, false); - - testConsumeInputStream(new byte[][]{ - createBytes(DEFAULT_RECORD_BYTE_COUNT * 2), - createBytes(DEFAULT_RECORD_BYTE_COUNT), - createBytes(DEFAULT_RECORD_BYTE_COUNT) - }, false); - - testConsumeInputStream(new byte[][]{ - createBytes(INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT) - }, false); - - testConsumeInputStream(new byte[][]{ - createBytes(INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT * 10), - }, false); - - testConsumeInputStream(new byte[][]{ - createBytes(INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT), - createBytes(INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT) - }, false); - - testConsumeInputStream(new byte[][]{ - createBytes(INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT), - createBytes(DEFAULT_RECORD_BYTE_COUNT), - createBytes(INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT) - }, false); + testConsumeInputStream(new byte[][] {createBytes(DEFAULT_RECORD_BYTE_COUNT)}, false); + + testConsumeInputStream( + new byte[][] { + createBytes(DEFAULT_RECORD_BYTE_COUNT), createBytes(DEFAULT_RECORD_BYTE_COUNT) + }, + false); + + testConsumeInputStream( + new byte[][] { + createBytes(DEFAULT_RECORD_BYTE_COUNT * 2), + createBytes(DEFAULT_RECORD_BYTE_COUNT), + createBytes(DEFAULT_RECORD_BYTE_COUNT) + }, + false); + + testConsumeInputStream( + new byte[][] {createBytes(INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT)}, false); + + testConsumeInputStream( + new byte[][] { + createBytes(INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT * 10), + }, + false); + + testConsumeInputStream( + new byte[][] { + createBytes(INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT), + createBytes(INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT) + }, + false); + + testConsumeInputStream( + new byte[][] { + createBytes(INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT), + createBytes(DEFAULT_RECORD_BYTE_COUNT), + createBytes(INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT) + }, + false); byte[][] testRecords = new byte[INITIAL_VALUE_ALLOCATION * 2][]; for (int i = 0; i < testRecords.length; i++) { @@ -113,5 +120,4 @@ public void testConsumeInputStream() throws IOException { } testConsumeInputStream(testRecords, false); } - } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcAliasToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcAliasToArrowTest.java index d32c2bbab91a8..e22686e890580 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcAliasToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcAliasToArrowTest.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.h2; import static org.apache.arrow.adapter.jdbc.AbstractJdbcToArrowTest.sqlToArrow; @@ -28,7 +27,6 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.List; - import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.VectorSchemaRoot; import org.apache.arrow.vector.types.pojo.Field; @@ -40,10 +38,8 @@ public class JdbcAliasToArrowTest { private Connection conn = null; - private static final String CREATE_STATEMENT = - "CREATE TABLE example_table (id INTEGER);"; - private static final String INSERT_STATEMENT = - "INSERT INTO example_table (id) VALUES (?);"; + private static final String CREATE_STATEMENT = "CREATE TABLE example_table (id INTEGER);"; + private static final String INSERT_STATEMENT = "INSERT INTO example_table (id) VALUES (?);"; private static final String QUERY = "SELECT id as a, id as b FROM example_table;"; private static final String DROP_STATEMENT = "DROP TABLE example_table;"; private static final String ORIGINAL_COLUMN_NAME = "ID"; @@ -62,10 +58,9 @@ public void setUp() throws Exception { } /** - * Test h2 database query with alias for column name and column label. - * To verify reading field alias from an H2 database works as expected. - * If this test fails, something is either wrong with the setup, - * or the H2 SQL behavior changed. + * Test h2 database query with alias for column name and column label. To verify reading field + * alias from an H2 database works as expected. If this test fails, something is either wrong with + * the setup, or the H2 SQL behavior changed. */ @Test public void testReadH2Alias() throws Exception { @@ -96,8 +91,8 @@ public void testReadH2Alias() throws Exception { } /** - * Test jdbc query results with alias to arrow works expected. - * Arrow result schema name should be field alias name. + * Test jdbc query results with alias to arrow works expected. Arrow result schema name should be + * field alias name. */ @Test public void testJdbcAliasToArrow() throws Exception { @@ -105,8 +100,7 @@ public void testJdbcAliasToArrow() throws Exception { insertRows(rowCount); try (ResultSet resultSet = conn.createStatement().executeQuery(QUERY)) { - final VectorSchemaRoot vector = - sqlToArrow(resultSet, new RootAllocator(Integer.MAX_VALUE)); + final VectorSchemaRoot vector = sqlToArrow(resultSet, new RootAllocator(Integer.MAX_VALUE)); assertEquals(rowCount, vector.getRowCount()); Schema vectorSchema = vector.getSchema(); diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowArrayTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowArrayTest.java index eabbdc5a25e5d..895dab52ca534 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowArrayTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowArrayTest.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.h2; import static org.apache.arrow.adapter.jdbc.AbstractJdbcToArrowTest.sqlToArrow; @@ -34,7 +33,6 @@ import java.sql.Types; import java.util.HashMap; import java.util.Map; - import org.apache.arrow.adapter.jdbc.JdbcFieldInfo; import org.apache.arrow.adapter.jdbc.JdbcToArrowConfig; import org.apache.arrow.adapter.jdbc.JdbcToArrowConfigBuilder; @@ -54,11 +52,12 @@ public class JdbcToArrowArrayTest { private Connection conn = null; private static final String CREATE_STATEMENT = - "CREATE TABLE array_table (id INTEGER, int_array INTEGER ARRAY, float_array REAL ARRAY, " + - "string_array VARCHAR ARRAY);"; + "CREATE TABLE array_table (id INTEGER, int_array INTEGER ARRAY, float_array REAL ARRAY, " + + "string_array VARCHAR ARRAY);"; private static final String INSERT_STATEMENT = "INSERT INTO array_table (id, int_array, float_array, string_array) VALUES (?, ?, ?, ?);"; - private static final String QUERY = "SELECT int_array, float_array, string_array FROM array_table ORDER BY id;"; + private static final String QUERY = + "SELECT int_array, float_array, string_array FROM array_table ORDER BY id;"; private static final String DROP_STATEMENT = "DROP TABLE array_table;"; private static Map arrayFieldMapping; @@ -158,7 +157,8 @@ public void testJdbcToArrow() throws Exception { insertRows(rowCount, intArrays, floatArrays, strArrays); final JdbcToArrowConfigBuilder builder = - new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), JdbcToArrowUtils.getUtcCalendar(), false); + new JdbcToArrowConfigBuilder( + new RootAllocator(Integer.MAX_VALUE), JdbcToArrowUtils.getUtcCalendar(), false); builder.setArraySubTypeByColumnNameMap(arrayFieldMapping); final JdbcToArrowConfig config = builder.build(); @@ -168,9 +168,12 @@ public void testJdbcToArrow() throws Exception { assertEquals(rowCount, vector.getRowCount()); - assertIntegerVectorEquals((ListVector) vector.getVector(INT_ARRAY_FIELD_NAME), rowCount, intArrays); - assertFloatVectorEquals((ListVector) vector.getVector(FLOAT_ARRAY_FIELD_NAME), rowCount, floatArrays); - assertStringVectorEquals((ListVector) vector.getVector(STRING_ARRAY_FIELD_NAME), rowCount, strArrays); + assertIntegerVectorEquals( + (ListVector) vector.getVector(INT_ARRAY_FIELD_NAME), rowCount, intArrays); + assertFloatVectorEquals( + (ListVector) vector.getVector(FLOAT_ARRAY_FIELD_NAME), rowCount, floatArrays); + assertStringVectorEquals( + (ListVector) vector.getVector(STRING_ARRAY_FIELD_NAME), rowCount, strArrays); } } @@ -179,30 +182,22 @@ public void testJdbcToArrowWithNulls() throws Exception { int rowCount = 4; Integer[][] intArrays = { - null, - {0}, - {1}, - {}, + null, {0}, {1}, {}, }; Float[][] floatArrays = { - { 2.0f }, - null, - { 3.0f }, - {}, + {2.0f}, null, {3.0f}, {}, }; String[][] stringArrays = { - {"4"}, - null, - {"5"}, - {}, + {"4"}, null, {"5"}, {}, }; insertRows(rowCount, intArrays, floatArrays, stringArrays); final JdbcToArrowConfigBuilder builder = - new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), JdbcToArrowUtils.getUtcCalendar(), false); + new JdbcToArrowConfigBuilder( + new RootAllocator(Integer.MAX_VALUE), JdbcToArrowUtils.getUtcCalendar(), false); builder.setArraySubTypeByColumnNameMap(arrayFieldMapping); final JdbcToArrowConfig config = builder.build(); @@ -212,13 +207,17 @@ public void testJdbcToArrowWithNulls() throws Exception { assertEquals(rowCount, vector.getRowCount()); - assertIntegerVectorEquals((ListVector) vector.getVector(INT_ARRAY_FIELD_NAME), rowCount, intArrays); - assertFloatVectorEquals((ListVector) vector.getVector(FLOAT_ARRAY_FIELD_NAME), rowCount, floatArrays); - assertStringVectorEquals((ListVector) vector.getVector(STRING_ARRAY_FIELD_NAME), rowCount, stringArrays); + assertIntegerVectorEquals( + (ListVector) vector.getVector(INT_ARRAY_FIELD_NAME), rowCount, intArrays); + assertFloatVectorEquals( + (ListVector) vector.getVector(FLOAT_ARRAY_FIELD_NAME), rowCount, floatArrays); + assertStringVectorEquals( + (ListVector) vector.getVector(STRING_ARRAY_FIELD_NAME), rowCount, stringArrays); } } - private void assertIntegerVectorEquals(ListVector listVector, int rowCount, Integer[][] expectedValues) { + private void assertIntegerVectorEquals( + ListVector listVector, int rowCount, Integer[][] expectedValues) { IntVector vector = (IntVector) listVector.getDataVector(); ArrowBuf offsetBuffer = listVector.getOffsetBuffer(); @@ -243,7 +242,8 @@ private void assertIntegerVectorEquals(ListVector listVector, int rowCount, Inte } } - private void assertFloatVectorEquals(ListVector listVector, int rowCount, Float[][] expectedValues) { + private void assertFloatVectorEquals( + ListVector listVector, int rowCount, Float[][] expectedValues) { Float4Vector vector = (Float4Vector) listVector.getDataVector(); ArrowBuf offsetBuffer = listVector.getOffsetBuffer(); @@ -268,7 +268,8 @@ private void assertFloatVectorEquals(ListVector listVector, int rowCount, Float[ } } - private void assertStringVectorEquals(ListVector listVector, int rowCount, String[][] expectedValues) { + private void assertStringVectorEquals( + ListVector listVector, int rowCount, String[][] expectedValues) { VarCharVector vector = (VarCharVector) listVector.getDataVector(); ArrowBuf offsetBuffer = listVector.getOffsetBuffer(); @@ -285,7 +286,8 @@ private void assertStringVectorEquals(ListVector listVector, int rowCount, Strin assertEquals(1, listVector.isSet(row)); assertEquals(expectedValues[row].length, offset - prevOffset); for (int i = prevOffset; i < offset; ++i) { - assertArrayEquals(expectedValues[row][i - prevOffset].getBytes(StandardCharsets.UTF_8), vector.get(i)); + assertArrayEquals( + expectedValues[row][i - prevOffset].getBytes(StandardCharsets.UTF_8), vector.get(i)); } prevOffset = offset; @@ -309,7 +311,7 @@ private Integer[][] generateIntegerArrayField(int numRows) { for (int i = 0; i < numRows; ++i) { int val = i * 4; - result[i] = new Integer[]{val, val + 1, val + 2, val + 3}; + result[i] = new Integer[] {val, val + 1, val + 2, val + 3}; } return result; @@ -317,10 +319,10 @@ private Integer[][] generateIntegerArrayField(int numRows) { private Float[][] generateFloatArrayField(int numRows) { Float[][] result = new Float[numRows][]; - + for (int i = 0; i < numRows; ++i) { int val = i * 4; - result[i] = new Float[]{(float) val, (float) val + 1, (float) val + 2, (float) val + 3}; + result[i] = new Float[] {(float) val, (float) val + 1, (float) val + 2, (float) val + 3}; } return result; @@ -331,22 +333,21 @@ private String[][] generateStringArrayField(int numRows) { for (int i = 0; i < numRows; ++i) { int val = i * 4; - result[i] = new String[]{ - String.valueOf(val), - String.valueOf(val + 1), - String.valueOf(val + 2), - String.valueOf(val + 3) }; + result[i] = + new String[] { + String.valueOf(val), + String.valueOf(val + 1), + String.valueOf(val + 2), + String.valueOf(val + 3) + }; } return result; } private void insertRows( - int numRows, - Integer[][] integerArrays, - Float[][] floatArrays, - String[][] strArrays) - throws SQLException { + int numRows, Integer[][] integerArrays, Float[][] floatArrays, String[][] strArrays) + throws SQLException { // Insert 4 Rows try (PreparedStatement stmt = conn.prepareStatement(INSERT_STATEMENT)) { diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowCharSetTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowCharSetTest.java index ab1b4b7fc2fea..14de2d6dc8f3c 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowCharSetTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowCharSetTest.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.h2; import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertVarcharVectorValues; @@ -29,7 +28,6 @@ import java.util.Arrays; import java.util.Calendar; import java.util.Collection; - import org.apache.arrow.adapter.jdbc.AbstractJdbcToArrowTest; import org.apache.arrow.adapter.jdbc.JdbcToArrowConfig; import org.apache.arrow.adapter.jdbc.JdbcToArrowConfigBuilder; @@ -47,8 +45,8 @@ import org.junit.runners.Parameterized.Parameters; /** - * JUnit Test Class which contains methods to test JDBC to Arrow data conversion functionality with UTF-8 Charset, - * including the multi-byte CJK characters for H2 database. + * JUnit Test Class which contains methods to test JDBC to Arrow data conversion functionality with + * UTF-8 Charset, including the multi-byte CJK characters for H2 database. */ @RunWith(Parameterized.class) public class JdbcToArrowCharSetTest extends AbstractJdbcToArrowTest { @@ -82,7 +80,7 @@ public void setUp() throws SQLException, ClassNotFoundException { String driver = "org.h2.Driver"; Class.forName(driver); conn = DriverManager.getConnection(url); - try (Statement stmt = conn.createStatement();) { + try (Statement stmt = conn.createStatement(); ) { stmt.executeUpdate(table.getCreate()); for (String insert : table.getData()) { stmt.executeUpdate(insert); @@ -99,39 +97,59 @@ public void setUp() throws SQLException, ClassNotFoundException { * @throws IOException on error */ @Parameters - public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { + public static Collection getTestData() + throws SQLException, ClassNotFoundException, IOException { return Arrays.asList(prepareTestData(testFiles, JdbcToArrowCharSetTest.class)); } /** - * Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes with UTF-8 Charset, including - * the multi-byte CJK characters. + * Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes with UTF-8 + * Charset, including the multi-byte CJK characters. */ @Test @Override public void testJdbcToArrowValues() throws SQLException, IOException { - testDataSets(sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE), - Calendar.getInstance()), false); + testDataSets( + sqlToArrow( + conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()), + false); testDataSets(sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE)), false); - testDataSets(sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), - new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()), false); + testDataSets( + sqlToArrow( + conn.createStatement().executeQuery(table.getQuery()), + new RootAllocator(Integer.MAX_VALUE), + Calendar.getInstance()), + false); testDataSets(sqlToArrow(conn.createStatement().executeQuery(table.getQuery())), false); - testDataSets(sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), - new RootAllocator(Integer.MAX_VALUE)), false); - testDataSets(sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), - Calendar.getInstance()), false); - testDataSets(sqlToArrow( - conn.createStatement().executeQuery(table.getQuery()), - new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()).build()), false); - testDataSets(sqlToArrow( - conn, - table.getQuery(), - new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()).build()), false); + testDataSets( + sqlToArrow( + conn.createStatement().executeQuery(table.getQuery()), + new RootAllocator(Integer.MAX_VALUE)), + false); + testDataSets( + sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), Calendar.getInstance()), + false); + testDataSets( + sqlToArrow( + conn.createStatement().executeQuery(table.getQuery()), + new JdbcToArrowConfigBuilder( + new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()) + .build()), + false); + testDataSets( + sqlToArrow( + conn, + table.getQuery(), + new JdbcToArrowConfigBuilder( + new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()) + .build()), + false); } @Test public void testJdbcSchemaMetadata() throws SQLException { - JdbcToArrowConfig config = new JdbcToArrowConfigBuilder(new RootAllocator(0), Calendar.getInstance(), true).build(); + JdbcToArrowConfig config = + new JdbcToArrowConfigBuilder(new RootAllocator(0), Calendar.getInstance(), true).build(); ResultSetMetaData rsmd = conn.createStatement().executeQuery(table.getQuery()).getMetaData(); Schema schema = JdbcToArrowUtils.jdbcToArrowSchema(rsmd, config); JdbcToArrowTestHelper.assertFieldMetadataMatchesResultSetMetadata(rsmd, schema); @@ -141,20 +159,26 @@ public void testJdbcSchemaMetadata() throws SQLException { * This method calls the assert methods for various DataSets. * * @param root VectorSchemaRoot for test - * @param isIncludeMapVector is this dataset checks includes map column. - * Jdbc type to 'map' mapping declared in configuration only manually + * @param isIncludeMapVector is this dataset checks includes map column. Jdbc type to 'map' + * mapping declared in configuration only manually */ @Override public void testDataSets(VectorSchemaRoot root, boolean isIncludeMapVector) { JdbcToArrowTestHelper.assertFieldMetadataIsEmpty(root); - assertVarcharVectorValues((VarCharVector) root.getVector(CLOB), table.getRowCount(), + assertVarcharVectorValues( + (VarCharVector) root.getVector(CLOB), + table.getRowCount(), getCharArrayWithCharSet(table.getValues(), CLOB, StandardCharsets.UTF_8)); - assertVarcharVectorValues((VarCharVector) root.getVector(VARCHAR), table.getRowCount(), + assertVarcharVectorValues( + (VarCharVector) root.getVector(VARCHAR), + table.getRowCount(), getCharArrayWithCharSet(table.getValues(), VARCHAR, StandardCharsets.UTF_8)); - assertVarcharVectorValues((VarCharVector) root.getVector(CHAR), table.getRowCount(), + assertVarcharVectorValues( + (VarCharVector) root.getVector(CHAR), + table.getRowCount(), getCharArrayWithCharSet(table.getValues(), CHAR, StandardCharsets.UTF_8)); } } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java index 54e7d5ffb27ed..d7c4be03b3542 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowDataTypesTest.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.h2; import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertBigIntVectorValues; @@ -40,7 +39,6 @@ import java.util.Arrays; import java.util.Calendar; import java.util.Collection; - import org.apache.arrow.adapter.jdbc.AbstractJdbcToArrowTest; import org.apache.arrow.adapter.jdbc.JdbcToArrowConfig; import org.apache.arrow.adapter.jdbc.JdbcToArrowConfigBuilder; @@ -71,8 +69,8 @@ import org.junit.runners.Parameterized.Parameters; /** - * JUnit Test Class which contains methods to test JDBC to Arrow data conversion functionality with various data types - * for H2 database using multiple test data files. + * JUnit Test Class which contains methods to test JDBC to Arrow data conversion functionality with + * various data types for H2 database using multiple test data files. */ @RunWith(Parameterized.class) public class JdbcToArrowDataTypesTest extends AbstractJdbcToArrowTest { @@ -137,43 +135,60 @@ public JdbcToArrowDataTypesTest(Table table) { * @throws IOException on error */ @Parameters - public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { + public static Collection getTestData() + throws SQLException, ClassNotFoundException, IOException { return Arrays.asList(prepareTestData(testFiles, JdbcToArrowDataTypesTest.class)); } - /** - * Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes. - */ + /** Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes. */ @Test @Override public void testJdbcToArrowValues() throws SQLException, IOException { - testDataSets(sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE), - Calendar.getInstance()), false); + testDataSets( + sqlToArrow( + conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()), + false); testDataSets(sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE)), false); - testDataSets(sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), - new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()), false); + testDataSets( + sqlToArrow( + conn.createStatement().executeQuery(table.getQuery()), + new RootAllocator(Integer.MAX_VALUE), + Calendar.getInstance()), + false); testDataSets(sqlToArrow(conn.createStatement().executeQuery(table.getQuery())), false); - testDataSets(sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), - new RootAllocator(Integer.MAX_VALUE)), false); - testDataSets(sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), Calendar.getInstance()), false); - testDataSets(sqlToArrow( - conn.createStatement().executeQuery(table.getQuery()), - new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()) - .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) - .build()), false); - testDataSets(sqlToArrow( - conn, - table.getQuery(), - new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()) - .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) - .build()), false); + testDataSets( + sqlToArrow( + conn.createStatement().executeQuery(table.getQuery()), + new RootAllocator(Integer.MAX_VALUE)), + false); + testDataSets( + sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), Calendar.getInstance()), + false); + testDataSets( + sqlToArrow( + conn.createStatement().executeQuery(table.getQuery()), + new JdbcToArrowConfigBuilder( + new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()) + .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) + .build()), + false); + testDataSets( + sqlToArrow( + conn, + table.getQuery(), + new JdbcToArrowConfigBuilder( + new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()) + .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) + .build()), + false); } @Test public void testJdbcSchemaMetadata() throws SQLException { - JdbcToArrowConfig config = new JdbcToArrowConfigBuilder(new RootAllocator(0), Calendar.getInstance(), true) - .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) - .build(); + JdbcToArrowConfig config = + new JdbcToArrowConfigBuilder(new RootAllocator(0), Calendar.getInstance(), true) + .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) + .build(); ResultSetMetaData rsmd = conn.createStatement().executeQuery(table.getQuery()).getMetaData(); Schema schema = JdbcToArrowUtils.jdbcToArrowSchema(rsmd, config); JdbcToArrowTestHelper.assertFieldMetadataMatchesResultSetMetadata(rsmd, schema); @@ -183,8 +198,8 @@ public void testJdbcSchemaMetadata() throws SQLException { * This method calls the assert methods for various DataSets. * * @param root VectorSchemaRoot for test - * @param isIncludeMapVector is this dataset checks includes map column. - * Jdbc type to 'map' mapping declared in configuration only manually + * @param isIncludeMapVector is this dataset checks includes map column. Jdbc type to 'map' + * mapping declared in configuration only manually */ @Override public void testDataSets(VectorSchemaRoot root, boolean isIncludeMapVector) { @@ -192,69 +207,99 @@ public void testDataSets(VectorSchemaRoot root, boolean isIncludeMapVector) { switch (table.getType()) { case BIGINT: - assertBigIntVectorValues((BigIntVector) root.getVector(table.getVector()), table.getValues().length, + assertBigIntVectorValues( + (BigIntVector) root.getVector(table.getVector()), + table.getValues().length, table.getLongValues()); break; case BINARY: case BLOB: - assertVarBinaryVectorValues((VarBinaryVector) root.getVector(table.getVector()), table.getValues().length, + assertVarBinaryVectorValues( + (VarBinaryVector) root.getVector(table.getVector()), + table.getValues().length, table.getBinaryValues()); break; case BIT: - assertBitVectorValues((BitVector) root.getVector(table.getVector()), table.getValues().length, + assertBitVectorValues( + (BitVector) root.getVector(table.getVector()), + table.getValues().length, table.getIntValues()); break; case BOOL: - assertBooleanVectorValues((BitVector) root.getVector(table.getVector()), table.getValues().length, + assertBooleanVectorValues( + (BitVector) root.getVector(table.getVector()), + table.getValues().length, table.getBoolValues()); break; case CHAR: case VARCHAR: case CLOB: - assertVarcharVectorValues((VarCharVector) root.getVector(table.getVector()), table.getValues().length, + assertVarcharVectorValues( + (VarCharVector) root.getVector(table.getVector()), + table.getValues().length, table.getCharValues()); break; case DATE: - assertDateVectorValues((DateDayVector) root.getVector(table.getVector()), table.getValues().length, + assertDateVectorValues( + (DateDayVector) root.getVector(table.getVector()), + table.getValues().length, table.getIntValues()); break; case TIME: - assertTimeVectorValues((TimeMilliVector) root.getVector(table.getVector()), table.getValues().length, + assertTimeVectorValues( + (TimeMilliVector) root.getVector(table.getVector()), + table.getValues().length, table.getLongValues()); break; case TIMESTAMP: - assertTimeStampVectorValues((TimeStampVector) root.getVector(table.getVector()), table.getValues().length, + assertTimeStampVectorValues( + (TimeStampVector) root.getVector(table.getVector()), + table.getValues().length, table.getLongValues()); break; case DECIMAL: - assertDecimalVectorValues((DecimalVector) root.getVector(table.getVector()), table.getValues().length, + assertDecimalVectorValues( + (DecimalVector) root.getVector(table.getVector()), + table.getValues().length, table.getBigDecimalValues()); break; case DOUBLE: - assertFloat8VectorValues((Float8Vector) root.getVector(table.getVector()), table.getValues().length, + assertFloat8VectorValues( + (Float8Vector) root.getVector(table.getVector()), + table.getValues().length, table.getDoubleValues()); break; case INT: - assertIntVectorValues((IntVector) root.getVector(table.getVector()), table.getValues().length, + assertIntVectorValues( + (IntVector) root.getVector(table.getVector()), + table.getValues().length, table.getIntValues()); break; case SMALLINT: - assertSmallIntVectorValues((SmallIntVector) root.getVector(table.getVector()), table.getValues().length, + assertSmallIntVectorValues( + (SmallIntVector) root.getVector(table.getVector()), + table.getValues().length, table.getIntValues()); break; case TINYINT: - assertTinyIntVectorValues((TinyIntVector) root.getVector(table.getVector()), table.getValues().length, + assertTinyIntVectorValues( + (TinyIntVector) root.getVector(table.getVector()), + table.getValues().length, table.getIntValues()); break; case REAL: - assertFloat4VectorValues((Float4Vector) root.getVector(table.getVector()), table.getValues().length, + assertFloat4VectorValues( + (Float4Vector) root.getVector(table.getVector()), + table.getValues().length, table.getFloatValues()); break; case NULL: assertNullVectorValues((NullVector) root.getVector(table.getVector()), table.getRowCount()); break; case LIST: - assertListVectorValues((ListVector) root.getVector(table.getVector()), table.getValues().length, + assertListVectorValues( + (ListVector) root.getVector(table.getVector()), + table.getValues().length, table.getListValues()); break; default: @@ -263,4 +308,3 @@ public void testDataSets(VectorSchemaRoot root, boolean isIncludeMapVector) { } } } - diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowMapDataTypeTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowMapDataTypeTest.java index a5d1ffa3f64de..8bb3812637acb 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowMapDataTypeTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowMapDataTypeTest.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.h2; import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertMapVectorValues; @@ -24,7 +23,6 @@ import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.util.Calendar; - import org.apache.arrow.adapter.jdbc.AbstractJdbcToArrowTest; import org.apache.arrow.adapter.jdbc.JdbcToArrowConfigBuilder; import org.apache.arrow.memory.RootAllocator; @@ -32,46 +30,48 @@ import org.apache.arrow.vector.complex.MapVector; import org.junit.Test; -/** - * Test MapConsumer with OTHER jdbc type. - */ +/** Test MapConsumer with OTHER jdbc type. */ public class JdbcToArrowMapDataTypeTest extends AbstractJdbcToArrowTest { public JdbcToArrowMapDataTypeTest() throws IOException { this.table = getTable("h2/test1_map_h2.yml", JdbcToArrowMapDataTypeTest.class); } - /** - * Test Method to test JdbcToArrow Functionality for Map form Types.OTHER column - */ + /** Test Method to test JdbcToArrow Functionality for Map form Types.OTHER column */ @Test @Override public void testJdbcToArrowValues() throws SQLException, IOException { Calendar calendar = Calendar.getInstance(); ResultSetMetaData rsmd = getQueryMetaData(table.getQuery()); - testDataSets(sqlToArrow( + testDataSets( + sqlToArrow( conn.createStatement().executeQuery(table.getQuery()), - new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()) - .setJdbcToArrowTypeConverter(jdbcToArrowTypeConverter(calendar, rsmd)) - .build()), true); - testDataSets(sqlToArrow( + new JdbcToArrowConfigBuilder( + new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()) + .setJdbcToArrowTypeConverter(jdbcToArrowTypeConverter(calendar, rsmd)) + .build()), + true); + testDataSets( + sqlToArrow( conn, table.getQuery(), - new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()) - .setJdbcToArrowTypeConverter(jdbcToArrowTypeConverter(calendar, rsmd)) - .build()), true); + new JdbcToArrowConfigBuilder( + new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()) + .setJdbcToArrowTypeConverter(jdbcToArrowTypeConverter(calendar, rsmd)) + .build()), + true); } /** * This method calls the assert methods for various DataSets. * * @param root VectorSchemaRoot for test - * @param isIncludeMapVector is this dataset checks includes map column. - * Jdbc type to 'map' mapping declared in configuration only manually + * @param isIncludeMapVector is this dataset checks includes map column. Jdbc type to 'map' + * mapping declared in configuration only manually */ @Override public void testDataSets(VectorSchemaRoot root, boolean isIncludeMapVector) { - assertMapVectorValues((MapVector) root.getVector(MAP), table.getRowCount(), - getMapValues(table.getValues(), MAP)); + assertMapVectorValues( + (MapVector) root.getVector(MAP), table.getRowCount(), getMapValues(table.getValues(), MAP)); } } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java index 31d32bd648906..51394764e385c 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowNullTest.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.h2; import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertBigIntVectorValues; @@ -51,7 +50,6 @@ import java.util.Arrays; import java.util.Calendar; import java.util.Collection; - import org.apache.arrow.adapter.jdbc.AbstractJdbcToArrowTest; import org.apache.arrow.adapter.jdbc.JdbcToArrowConfig; import org.apache.arrow.adapter.jdbc.JdbcToArrowConfigBuilder; @@ -82,8 +80,8 @@ import org.junit.runners.Parameterized.Parameters; /** - * JUnit Test Class which contains methods to test JDBC to Arrow data conversion functionality with null values for - * H2 database. + * JUnit Test Class which contains methods to test JDBC to Arrow data conversion functionality with + * null values for H2 database. */ @RunWith(Parameterized.class) public class JdbcToArrowNullTest extends AbstractJdbcToArrowTest { @@ -116,47 +114,67 @@ public JdbcToArrowNullTest(Table table) { * @throws IOException on error */ @Parameters - public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { + public static Collection getTestData() + throws SQLException, ClassNotFoundException, IOException { return Arrays.asList(prepareTestData(testFiles, JdbcToArrowNullTest.class)); } /** - * Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes with null values. + * Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes with null + * values. */ @Test @Override public void testJdbcToArrowValues() throws SQLException, IOException { - testDataSets(sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE), - Calendar.getInstance()), false); + testDataSets( + sqlToArrow( + conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()), + false); testDataSets(sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE)), false); - testDataSets(sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), - new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()), false); + testDataSets( + sqlToArrow( + conn.createStatement().executeQuery(table.getQuery()), + new RootAllocator(Integer.MAX_VALUE), + Calendar.getInstance()), + false); testDataSets(sqlToArrow(conn.createStatement().executeQuery(table.getQuery())), false); - testDataSets(sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), - new RootAllocator(Integer.MAX_VALUE)), false); - testDataSets(sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), Calendar.getInstance()), false); + testDataSets( + sqlToArrow( + conn.createStatement().executeQuery(table.getQuery()), + new RootAllocator(Integer.MAX_VALUE)), + false); + testDataSets( + sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), Calendar.getInstance()), + false); Calendar calendar = Calendar.getInstance(); ResultSetMetaData rsmd = getQueryMetaData(table.getQuery()); - testDataSets(sqlToArrow( - conn.createStatement().executeQuery(table.getQuery()), - new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()) - .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) - .setJdbcToArrowTypeConverter(jdbcToArrowTypeConverter(calendar, rsmd)) - .build()), true); - testDataSets(sqlToArrow( - conn, - table.getQuery(), - new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()) - .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) - .setJdbcToArrowTypeConverter(jdbcToArrowTypeConverter(calendar, rsmd)) - .build()), true); + testDataSets( + sqlToArrow( + conn.createStatement().executeQuery(table.getQuery()), + new JdbcToArrowConfigBuilder( + new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()) + .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) + .setJdbcToArrowTypeConverter(jdbcToArrowTypeConverter(calendar, rsmd)) + .build()), + true); + testDataSets( + sqlToArrow( + conn, + table.getQuery(), + new JdbcToArrowConfigBuilder( + new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()) + .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) + .setJdbcToArrowTypeConverter(jdbcToArrowTypeConverter(calendar, rsmd)) + .build()), + true); } @Test public void testJdbcSchemaMetadata() throws SQLException { - JdbcToArrowConfig config = new JdbcToArrowConfigBuilder(new RootAllocator(0), Calendar.getInstance(), true) - .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) - .build(); + JdbcToArrowConfig config = + new JdbcToArrowConfigBuilder(new RootAllocator(0), Calendar.getInstance(), true) + .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) + .build(); ResultSetMetaData rsmd = conn.createStatement().executeQuery(table.getQuery()).getMetaData(); Schema schema = JdbcToArrowUtils.jdbcToArrowSchema(rsmd, config); JdbcToArrowTestHelper.assertFieldMetadataMatchesResultSetMetadata(rsmd, schema); @@ -166,8 +184,8 @@ public void testJdbcSchemaMetadata() throws SQLException { * This method calls the assert methods for various DataSets. * * @param root VectorSchemaRoot for test - * @param isIncludeMapVector is this dataset checks includes map column. - * Jdbc type to 'map' mapping declared in configuration only manually + * @param isIncludeMapVector is this dataset checks includes map column. Jdbc type to 'map' + * mapping declared in configuration only manually */ @Override public void testDataSets(VectorSchemaRoot root, boolean isIncludeMapVector) { @@ -178,7 +196,8 @@ public void testDataSets(VectorSchemaRoot root, boolean isIncludeMapVector) { sqlToArrowTestNullValues(table.getVectors(), root, table.getRowCount()); break; case SELECTED_NULL_COLUMN: - sqlToArrowTestSelectedNullColumnsValues(table.getVectors(), root, table.getRowCount(), isIncludeMapVector); + sqlToArrowTestSelectedNullColumnsValues( + table.getVectors(), root, table.getRowCount(), isIncludeMapVector); break; case SELECTED_NULL_ROW: testAllVectorValues(root, isIncludeMapVector); @@ -192,62 +211,96 @@ public void testDataSets(VectorSchemaRoot root, boolean isIncludeMapVector) { private void testAllVectorValues(VectorSchemaRoot root, boolean isIncludeMapVector) { JdbcToArrowTestHelper.assertFieldMetadataIsEmpty(root); - assertBigIntVectorValues((BigIntVector) root.getVector(BIGINT), table.getRowCount(), + assertBigIntVectorValues( + (BigIntVector) root.getVector(BIGINT), + table.getRowCount(), getLongValues(table.getValues(), BIGINT)); - assertTinyIntVectorValues((TinyIntVector) root.getVector(TINYINT), table.getRowCount(), + assertTinyIntVectorValues( + (TinyIntVector) root.getVector(TINYINT), + table.getRowCount(), getIntValues(table.getValues(), TINYINT)); - assertSmallIntVectorValues((SmallIntVector) root.getVector(SMALLINT), table.getRowCount(), + assertSmallIntVectorValues( + (SmallIntVector) root.getVector(SMALLINT), + table.getRowCount(), getIntValues(table.getValues(), SMALLINT)); - assertVarBinaryVectorValues((VarBinaryVector) root.getVector(BINARY), table.getRowCount(), + assertVarBinaryVectorValues( + (VarBinaryVector) root.getVector(BINARY), + table.getRowCount(), getBinaryValues(table.getValues(), BINARY)); - assertVarBinaryVectorValues((VarBinaryVector) root.getVector(BLOB), table.getRowCount(), + assertVarBinaryVectorValues( + (VarBinaryVector) root.getVector(BLOB), + table.getRowCount(), getBinaryValues(table.getValues(), BLOB)); - assertVarcharVectorValues((VarCharVector) root.getVector(CLOB), table.getRowCount(), + assertVarcharVectorValues( + (VarCharVector) root.getVector(CLOB), + table.getRowCount(), getCharArray(table.getValues(), CLOB)); - assertVarcharVectorValues((VarCharVector) root.getVector(VARCHAR), table.getRowCount(), + assertVarcharVectorValues( + (VarCharVector) root.getVector(VARCHAR), + table.getRowCount(), getCharArray(table.getValues(), VARCHAR)); - assertVarcharVectorValues((VarCharVector) root.getVector(CHAR), table.getRowCount(), + assertVarcharVectorValues( + (VarCharVector) root.getVector(CHAR), + table.getRowCount(), getCharArray(table.getValues(), CHAR)); - assertIntVectorValues((IntVector) root.getVector(INT), table.getRowCount(), - getIntValues(table.getValues(), INT)); + assertIntVectorValues( + (IntVector) root.getVector(INT), table.getRowCount(), getIntValues(table.getValues(), INT)); - assertBitVectorValues((BitVector) root.getVector(BIT), table.getRowCount(), - getIntValues(table.getValues(), BIT)); + assertBitVectorValues( + (BitVector) root.getVector(BIT), table.getRowCount(), getIntValues(table.getValues(), BIT)); - assertBooleanVectorValues((BitVector) root.getVector(BOOL), table.getRowCount(), + assertBooleanVectorValues( + (BitVector) root.getVector(BOOL), + table.getRowCount(), getBooleanValues(table.getValues(), BOOL)); - assertDateVectorValues((DateDayVector) root.getVector(DATE), table.getRowCount(), + assertDateVectorValues( + (DateDayVector) root.getVector(DATE), + table.getRowCount(), getIntValues(table.getValues(), DATE)); - assertTimeVectorValues((TimeMilliVector) root.getVector(TIME), table.getRowCount(), + assertTimeVectorValues( + (TimeMilliVector) root.getVector(TIME), + table.getRowCount(), getLongValues(table.getValues(), TIME)); - assertTimeStampVectorValues((TimeStampVector) root.getVector(TIMESTAMP), table.getRowCount(), + assertTimeStampVectorValues( + (TimeStampVector) root.getVector(TIMESTAMP), + table.getRowCount(), getLongValues(table.getValues(), TIMESTAMP)); - assertDecimalVectorValues((DecimalVector) root.getVector(DECIMAL), table.getRowCount(), + assertDecimalVectorValues( + (DecimalVector) root.getVector(DECIMAL), + table.getRowCount(), getDecimalValues(table.getValues(), DECIMAL)); - assertFloat8VectorValues((Float8Vector) root.getVector(DOUBLE), table.getRowCount(), + assertFloat8VectorValues( + (Float8Vector) root.getVector(DOUBLE), + table.getRowCount(), getDoubleValues(table.getValues(), DOUBLE)); - assertFloat4VectorValues((Float4Vector) root.getVector(REAL), table.getRowCount(), + assertFloat4VectorValues( + (Float4Vector) root.getVector(REAL), + table.getRowCount(), getFloatValues(table.getValues(), REAL)); - assertListVectorValues((ListVector) root.getVector(LIST), table.getRowCount(), + assertListVectorValues( + (ListVector) root.getVector(LIST), + table.getRowCount(), getListValues(table.getValues(), LIST)); if (isIncludeMapVector) { - assertMapVectorValues((MapVector) root.getVector(MAP), table.getRowCount(), - getMapValues(table.getValues(), MAP)); + assertMapVectorValues( + (MapVector) root.getVector(MAP), + table.getRowCount(), + getMapValues(table.getValues(), MAP)); } } @@ -285,11 +338,11 @@ public void sqlToArrowTestNullValues(String[] vectors, VectorSchemaRoot root, in * @param vectors Vectors to test * @param root VectorSchemaRoot for test * @param rowCount number of rows - * @param isIncludeMapVector is this dataset checks includes map column. - * Jdbc type to 'map' mapping declared in configuration only manually + * @param isIncludeMapVector is this dataset checks includes map column. Jdbc type to 'map' + * mapping declared in configuration only manually */ - public void sqlToArrowTestSelectedNullColumnsValues(String[] vectors, VectorSchemaRoot root, int rowCount, - boolean isIncludeMapVector) { + public void sqlToArrowTestSelectedNullColumnsValues( + String[] vectors, VectorSchemaRoot root, int rowCount, boolean isIncludeMapVector) { assertNullValues((BigIntVector) root.getVector(vectors[0]), rowCount); assertNullValues((DecimalVector) root.getVector(vectors[1]), rowCount); assertNullValues((Float8Vector) root.getVector(vectors[2]), rowCount); @@ -308,5 +361,4 @@ public void sqlToArrowTestSelectedNullColumnsValues(String[] vectors, VectorSche assertNullValues((MapVector) root.getVector(vectors[14]), rowCount); } } - } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowOptionalColumnsTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowOptionalColumnsTest.java index 4d0bbfc7a993c..47713d9099da6 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowOptionalColumnsTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowOptionalColumnsTest.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.h2; import static junit.framework.TestCase.assertTrue; @@ -24,7 +23,6 @@ import java.sql.SQLException; import java.util.Arrays; import java.util.Collection; - import org.apache.arrow.adapter.jdbc.AbstractJdbcToArrowTest; import org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper; import org.apache.arrow.adapter.jdbc.Table; @@ -40,9 +38,7 @@ */ @RunWith(Parameterized.class) public class JdbcToArrowOptionalColumnsTest extends AbstractJdbcToArrowTest { - private static final String[] testFiles = { - "h2/test1_null_and_notnull.yml" - }; + private static final String[] testFiles = {"h2/test1_null_and_notnull.yml"}; /** * Constructor which populates the table object for each test iteration. @@ -57,17 +53,19 @@ public JdbcToArrowOptionalColumnsTest(Table table) { * Get the test data as a collection of Table objects for each test iteration. * * @return Collection of Table objects - * @throws SQLException on error + * @throws SQLException on error * @throws ClassNotFoundException on error - * @throws IOException on error + * @throws IOException on error */ @Parameterized.Parameters - public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { + public static Collection getTestData() + throws SQLException, ClassNotFoundException, IOException { return Arrays.asList(prepareTestData(testFiles, JdbcToArrowOptionalColumnsTest.class)); } /** - * Test Method to test JdbcToArrow Functionality for dealing with nullable and non-nullable columns. + * Test Method to test JdbcToArrow Functionality for dealing with nullable and non-nullable + * columns. */ @Test @Override @@ -76,12 +74,13 @@ public void testJdbcToArrowValues() throws SQLException, IOException { } /** - * This method calls the assert methods for various DataSets. We verify that a SQL `NULL` column becomes - * nullable in the VectorSchemaRoot, and that a SQL `NOT NULL` column becomes non-nullable. + * This method calls the assert methods for various DataSets. We verify that a SQL `NULL` column + * becomes nullable in the VectorSchemaRoot, and that a SQL `NOT NULL` column becomes + * non-nullable. * * @param root VectorSchemaRoot for test - * @param isIncludeMapVector is this dataset checks includes map column. - * Jdbc type to 'map' mapping declared in configuration only manually + * @param isIncludeMapVector is this dataset checks includes map column. Jdbc type to 'map' + * mapping declared in configuration only manually */ @Override public void testDataSets(VectorSchemaRoot root, boolean isIncludeMapVector) { @@ -90,5 +89,4 @@ public void testDataSets(VectorSchemaRoot root, boolean isIncludeMapVector) { assertTrue(root.getSchema().getFields().get(0).isNullable()); assertFalse(root.getSchema().getFields().get(1).isNullable()); } - } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java index a925dd7ee32a8..d290b9bf08960 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTest.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.h2; import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.*; @@ -29,7 +28,6 @@ import java.util.Collection; import java.util.stream.Collectors; import java.util.stream.Stream; - import org.apache.arrow.adapter.jdbc.AbstractJdbcToArrowTest; import org.apache.arrow.adapter.jdbc.ArrowVectorIterator; import org.apache.arrow.adapter.jdbc.JdbcToArrow; @@ -64,8 +62,8 @@ import org.junit.runners.Parameterized; /** - * JUnit Test Class which contains methods to test JDBC to Arrow data conversion functionality with various data types - * for H2 database using single test data file. + * JUnit Test Class which contains methods to test JDBC to Arrow data conversion functionality with + * various data types for H2 database using single test data file. */ @RunWith(Parameterized.class) public class JdbcToArrowTest extends AbstractJdbcToArrowTest { @@ -92,53 +90,72 @@ public JdbcToArrowTest(Table table, boolean reuseVectorSchemaRoot) { * @throws IOException on error */ @Parameterized.Parameters(name = "table = {0}, reuse batch = {1}") - public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { - return Arrays.stream(prepareTestData(testFiles, JdbcToArrowTest.class)).flatMap(row -> - Stream.of(new Object[] {row[0], true}, new Object[] {row[0], false})).collect(Collectors.toList()); + public static Collection getTestData() + throws SQLException, ClassNotFoundException, IOException { + return Arrays.stream(prepareTestData(testFiles, JdbcToArrowTest.class)) + .flatMap(row -> Stream.of(new Object[] {row[0], true}, new Object[] {row[0], false})) + .collect(Collectors.toList()); } /** - * Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes with only one test data file. + * Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes with only one + * test data file. */ @Test @Override public void testJdbcToArrowValues() throws SQLException, IOException { - testDataSets(sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE), - Calendar.getInstance()), false); + testDataSets( + sqlToArrow( + conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()), + false); testDataSets(sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE)), false); - testDataSets(sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), - new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()), false); + testDataSets( + sqlToArrow( + conn.createStatement().executeQuery(table.getQuery()), + new RootAllocator(Integer.MAX_VALUE), + Calendar.getInstance()), + false); testDataSets(sqlToArrow(conn.createStatement().executeQuery(table.getQuery())), false); - testDataSets(sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), - new RootAllocator(Integer.MAX_VALUE)), false); - testDataSets(sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), - Calendar.getInstance()), false); + testDataSets( + sqlToArrow( + conn.createStatement().executeQuery(table.getQuery()), + new RootAllocator(Integer.MAX_VALUE)), + false); + testDataSets( + sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), Calendar.getInstance()), + false); Calendar calendar = Calendar.getInstance(); ResultSetMetaData rsmd = getQueryMetaData(table.getQuery()); - testDataSets(sqlToArrow( - conn.createStatement().executeQuery(table.getQuery()), - new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()) - .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) - .setJdbcToArrowTypeConverter(jdbcToArrowTypeConverter(calendar, rsmd)) - .build()), true); - testDataSets(sqlToArrow( - conn, - table.getQuery(), - new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), calendar) - .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) - .setJdbcToArrowTypeConverter(jdbcToArrowTypeConverter(calendar, rsmd)) - .build()), true); + testDataSets( + sqlToArrow( + conn.createStatement().executeQuery(table.getQuery()), + new JdbcToArrowConfigBuilder( + new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()) + .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) + .setJdbcToArrowTypeConverter(jdbcToArrowTypeConverter(calendar, rsmd)) + .build()), + true); + testDataSets( + sqlToArrow( + conn, + table.getQuery(), + new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), calendar) + .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) + .setJdbcToArrowTypeConverter(jdbcToArrowTypeConverter(calendar, rsmd)) + .build()), + true); } @Test public void testJdbcSchemaMetadata() throws SQLException { Calendar calendar = Calendar.getInstance(); ResultSetMetaData rsmd = getQueryMetaData(table.getQuery()); - JdbcToArrowConfig config = new JdbcToArrowConfigBuilder(new RootAllocator(0), calendar, true) - .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) - .setJdbcToArrowTypeConverter(jdbcToArrowTypeConverter(calendar, rsmd)) - .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) - .build(); + JdbcToArrowConfig config = + new JdbcToArrowConfigBuilder(new RootAllocator(0), calendar, true) + .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) + .setJdbcToArrowTypeConverter(jdbcToArrowTypeConverter(calendar, rsmd)) + .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) + .build(); Schema schema = JdbcToArrowUtils.jdbcToArrowSchema(rsmd, config); JdbcToArrowTestHelper.assertFieldMetadataMatchesResultSetMetadata(rsmd, schema); } @@ -147,71 +164,105 @@ public void testJdbcSchemaMetadata() throws SQLException { * This method calls the assert methods for various DataSets. * * @param root VectorSchemaRoot for test - * @param isIncludeMapVector is this dataset checks includes map column. - * Jdbc type to 'map' mapping declared in configuration only manually + * @param isIncludeMapVector is this dataset checks includes map column. Jdbc type to 'map' + * mapping declared in configuration only manually */ @Override public void testDataSets(VectorSchemaRoot root, boolean isIncludeMapVector) { JdbcToArrowTestHelper.assertFieldMetadataIsEmpty(root); - assertBigIntVectorValues((BigIntVector) root.getVector(BIGINT), table.getRowCount(), + assertBigIntVectorValues( + (BigIntVector) root.getVector(BIGINT), + table.getRowCount(), getLongValues(table.getValues(), BIGINT)); - assertTinyIntVectorValues((TinyIntVector) root.getVector(TINYINT), table.getRowCount(), + assertTinyIntVectorValues( + (TinyIntVector) root.getVector(TINYINT), + table.getRowCount(), getIntValues(table.getValues(), TINYINT)); - assertSmallIntVectorValues((SmallIntVector) root.getVector(SMALLINT), table.getRowCount(), + assertSmallIntVectorValues( + (SmallIntVector) root.getVector(SMALLINT), + table.getRowCount(), getIntValues(table.getValues(), SMALLINT)); - assertVarBinaryVectorValues((VarBinaryVector) root.getVector(BINARY), table.getRowCount(), + assertVarBinaryVectorValues( + (VarBinaryVector) root.getVector(BINARY), + table.getRowCount(), getBinaryValues(table.getValues(), BINARY)); - assertVarBinaryVectorValues((VarBinaryVector) root.getVector(BLOB), table.getRowCount(), + assertVarBinaryVectorValues( + (VarBinaryVector) root.getVector(BLOB), + table.getRowCount(), getBinaryValues(table.getValues(), BLOB)); - assertVarcharVectorValues((VarCharVector) root.getVector(CLOB), table.getRowCount(), + assertVarcharVectorValues( + (VarCharVector) root.getVector(CLOB), + table.getRowCount(), getCharArray(table.getValues(), CLOB)); - assertVarcharVectorValues((VarCharVector) root.getVector(VARCHAR), table.getRowCount(), + assertVarcharVectorValues( + (VarCharVector) root.getVector(VARCHAR), + table.getRowCount(), getCharArray(table.getValues(), VARCHAR)); - assertVarcharVectorValues((VarCharVector) root.getVector(CHAR), table.getRowCount(), + assertVarcharVectorValues( + (VarCharVector) root.getVector(CHAR), + table.getRowCount(), getCharArray(table.getValues(), CHAR)); - assertIntVectorValues((IntVector) root.getVector(INT), table.getRowCount(), - getIntValues(table.getValues(), INT)); + assertIntVectorValues( + (IntVector) root.getVector(INT), table.getRowCount(), getIntValues(table.getValues(), INT)); - assertBitVectorValues((BitVector) root.getVector(BIT), table.getRowCount(), - getIntValues(table.getValues(), BIT)); + assertBitVectorValues( + (BitVector) root.getVector(BIT), table.getRowCount(), getIntValues(table.getValues(), BIT)); - assertBooleanVectorValues((BitVector) root.getVector(BOOL), table.getRowCount(), + assertBooleanVectorValues( + (BitVector) root.getVector(BOOL), + table.getRowCount(), getBooleanValues(table.getValues(), BOOL)); - assertDateVectorValues((DateDayVector) root.getVector(DATE), table.getRowCount(), + assertDateVectorValues( + (DateDayVector) root.getVector(DATE), + table.getRowCount(), getIntValues(table.getValues(), DATE)); - assertTimeVectorValues((TimeMilliVector) root.getVector(TIME), table.getRowCount(), + assertTimeVectorValues( + (TimeMilliVector) root.getVector(TIME), + table.getRowCount(), getLongValues(table.getValues(), TIME)); - assertTimeStampVectorValues((TimeStampVector) root.getVector(TIMESTAMP), table.getRowCount(), + assertTimeStampVectorValues( + (TimeStampVector) root.getVector(TIMESTAMP), + table.getRowCount(), getLongValues(table.getValues(), TIMESTAMP)); - assertDecimalVectorValues((DecimalVector) root.getVector(DECIMAL), table.getRowCount(), + assertDecimalVectorValues( + (DecimalVector) root.getVector(DECIMAL), + table.getRowCount(), getDecimalValues(table.getValues(), DECIMAL)); - assertFloat8VectorValues((Float8Vector) root.getVector(DOUBLE), table.getRowCount(), + assertFloat8VectorValues( + (Float8Vector) root.getVector(DOUBLE), + table.getRowCount(), getDoubleValues(table.getValues(), DOUBLE)); - assertFloat4VectorValues((Float4Vector) root.getVector(REAL), table.getRowCount(), + assertFloat4VectorValues( + (Float4Vector) root.getVector(REAL), + table.getRowCount(), getFloatValues(table.getValues(), REAL)); assertNullVectorValues((NullVector) root.getVector(NULL), table.getRowCount()); - assertListVectorValues((ListVector) root.getVector(LIST), table.getRowCount(), + assertListVectorValues( + (ListVector) root.getVector(LIST), + table.getRowCount(), getListValues(table.getValues(), LIST)); if (isIncludeMapVector) { - assertMapVectorValues((MapVector) root.getVector(MAP), table.getRowCount(), - getMapValues(table.getValues(), MAP)); + assertMapVectorValues( + (MapVector) root.getVector(MAP), + table.getRowCount(), + getMapValues(table.getValues(), MAP)); } } @@ -221,11 +272,12 @@ public void runLargeNumberOfRows() throws IOException, SQLException { int x = 0; final int targetRows = 600000; ResultSet rs = ResultSetUtility.generateBasicResultSet(targetRows); - JdbcToArrowConfig config = new JdbcToArrowConfigBuilder( - allocator, JdbcToArrowUtils.getUtcCalendar(), /* include metadata */ false) - .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) - .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) - .build(); + JdbcToArrowConfig config = + new JdbcToArrowConfigBuilder( + allocator, JdbcToArrowUtils.getUtcCalendar(), /* include metadata */ false) + .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) + .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) + .build(); try (ArrowVectorIterator iter = JdbcToArrow.sqlToArrowVectorIterator(rs, config)) { while (iter.hasNext()) { diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java index fe08db161c8ac..c4930c3ab6017 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowTimeZoneTest.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.h2; import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.assertDateVectorValues; @@ -28,7 +27,6 @@ import java.util.Calendar; import java.util.Collection; import java.util.TimeZone; - import org.apache.arrow.adapter.jdbc.AbstractJdbcToArrowTest; import org.apache.arrow.adapter.jdbc.JdbcToArrowConfig; import org.apache.arrow.adapter.jdbc.JdbcToArrowConfigBuilder; @@ -47,10 +45,9 @@ import org.junit.runners.Parameterized.Parameters; /** - * JUnit Test Class which contains methods to test JDBC to Arrow data conversion functionality with TimeZone based Date, - * Time and Timestamp datatypes for H2 database. + * JUnit Test Class which contains methods to test JDBC to Arrow data conversion functionality with + * TimeZone based Date, Time and Timestamp datatypes for H2 database. */ - @RunWith(Parameterized.class) public class JdbcToArrowTimeZoneTest extends AbstractJdbcToArrowTest { @@ -94,40 +91,60 @@ public JdbcToArrowTimeZoneTest(Table table) { * @throws IOException on error */ @Parameters - public static Collection getTestData() throws SQLException, ClassNotFoundException, IOException { + public static Collection getTestData() + throws SQLException, ClassNotFoundException, IOException { return Arrays.asList(prepareTestData(testFiles, JdbcToArrowTimeZoneTest.class)); } /** - * Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes with TimeZone based Date, - * Time and Timestamp datatype. + * Test Method to test JdbcToArrow Functionality for various H2 DB based datatypes with TimeZone + * based Date, Time and Timestamp datatype. */ @Test @Override public void testJdbcToArrowValues() throws SQLException, IOException { - testDataSets(sqlToArrow(conn, table.getQuery(), new RootAllocator(Integer.MAX_VALUE), - Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone()))), false); - testDataSets(sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), - new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone()))), false); - testDataSets(sqlToArrow(conn.createStatement().executeQuery(table.getQuery()), - Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone()))), false); - testDataSets(sqlToArrow( - conn.createStatement().executeQuery(table.getQuery()), - new JdbcToArrowConfigBuilder( + testDataSets( + sqlToArrow( + conn, + table.getQuery(), new RootAllocator(Integer.MAX_VALUE), - Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone()))).build()), false); - testDataSets(sqlToArrow( - conn, - table.getQuery(), - new JdbcToArrowConfigBuilder( + Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone()))), + false); + testDataSets( + sqlToArrow( + conn.createStatement().executeQuery(table.getQuery()), new RootAllocator(Integer.MAX_VALUE), - Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone()))).build()), false); + Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone()))), + false); + testDataSets( + sqlToArrow( + conn.createStatement().executeQuery(table.getQuery()), + Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone()))), + false); + testDataSets( + sqlToArrow( + conn.createStatement().executeQuery(table.getQuery()), + new JdbcToArrowConfigBuilder( + new RootAllocator(Integer.MAX_VALUE), + Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone()))) + .build()), + false); + testDataSets( + sqlToArrow( + conn, + table.getQuery(), + new JdbcToArrowConfigBuilder( + new RootAllocator(Integer.MAX_VALUE), + Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone()))) + .build()), + false); } @Test public void testJdbcSchemaMetadata() throws SQLException { Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone(table.getTimezone())); - JdbcToArrowConfig config = new JdbcToArrowConfigBuilder(new RootAllocator(0), calendar, true).build(); + JdbcToArrowConfig config = + new JdbcToArrowConfigBuilder(new RootAllocator(0), calendar, true).build(); ResultSetMetaData rsmd = conn.createStatement().executeQuery(table.getQuery()).getMetaData(); Schema schema = JdbcToArrowUtils.jdbcToArrowSchema(rsmd, config); JdbcToArrowTestHelper.assertFieldMetadataMatchesResultSetMetadata(rsmd, schema); @@ -137,8 +154,8 @@ public void testJdbcSchemaMetadata() throws SQLException { * This method calls the assert methods for various DataSets. * * @param root VectorSchemaRoot for test - * @param isIncludeMapVector is this dataset checks includes map column. - * Jdbc type to 'map' mapping declared in configuration only manually + * @param isIncludeMapVector is this dataset checks includes map column. Jdbc type to 'map' + * mapping declared in configuration only manually */ @Override public void testDataSets(VectorSchemaRoot root, boolean isIncludeMapVector) { @@ -148,19 +165,25 @@ public void testDataSets(VectorSchemaRoot root, boolean isIncludeMapVector) { case EST_DATE: case GMT_DATE: case PST_DATE: - assertDateVectorValues((DateDayVector) root.getVector(table.getVector()), table.getValues().length, + assertDateVectorValues( + (DateDayVector) root.getVector(table.getVector()), + table.getValues().length, table.getIntValues()); break; case EST_TIME: case GMT_TIME: case PST_TIME: - assertTimeVectorValues((TimeMilliVector) root.getVector(table.getVector()), table.getValues().length, + assertTimeVectorValues( + (TimeMilliVector) root.getVector(table.getVector()), + table.getValues().length, table.getLongValues()); break; case EST_TIMESTAMP: case GMT_TIMESTAMP: case PST_TIMESTAMP: - assertTimeStampVectorValues((TimeStampVector) root.getVector(table.getVector()), table.getValues().length, + assertTimeStampVectorValues( + (TimeStampVector) root.getVector(table.getVector()), + table.getValues().length, table.getLongValues()); break; default: @@ -168,5 +191,4 @@ public void testDataSets(VectorSchemaRoot root, boolean isIncludeMapVector) { break; } } - } diff --git a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowVectorIteratorTest.java b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowVectorIteratorTest.java index 1d7e2760f843e..caa1c1d971adb 100644 --- a/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowVectorIteratorTest.java +++ b/java/adapter/jdbc/src/test/java/org/apache/arrow/adapter/jdbc/h2/JdbcToArrowVectorIteratorTest.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.jdbc.h2; import static org.apache.arrow.adapter.jdbc.JdbcToArrowTestHelper.getBinaryValues; @@ -42,7 +41,6 @@ import java.util.Arrays; import java.util.Calendar; import java.util.List; - import org.apache.arrow.adapter.jdbc.ArrowVectorIterator; import org.apache.arrow.adapter.jdbc.JdbcToArrow; import org.apache.arrow.adapter.jdbc.JdbcToArrowConfig; @@ -91,14 +89,15 @@ public JdbcToArrowVectorIteratorTest(Table table, boolean reuseVectorSchemaRoot) @Test @Override public void testJdbcToArrowValues() throws SQLException, IOException { - JdbcToArrowConfig config = new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), - Calendar.getInstance()) - .setTargetBatchSize(3) - .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) - .build(); + JdbcToArrowConfig config = + new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()) + .setTargetBatchSize(3) + .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) + .build(); ArrowVectorIterator iterator = - JdbcToArrow.sqlToArrowVectorIterator(conn.createStatement().executeQuery(table.getQuery()), config); + JdbcToArrow.sqlToArrowVectorIterator( + conn.createStatement().executeQuery(table.getQuery()), config); validate(iterator); } @@ -106,27 +105,28 @@ public void testJdbcToArrowValues() throws SQLException, IOException { @Test public void testVectorSchemaRootReuse() throws SQLException, IOException { Integer[][] intValues = { - {101, 102, 103}, - {104, null, null}, - {107, 108, 109}, - {110} + {101, 102, 103}, + {104, null, null}, + {107, 108, 109}, + {110} }; Integer[][][] listValues = { - {{1, 2, 3}, {1, 2}, {1}}, - {{2, 3, 4}, {2, 3}, {2}}, - {{3, 4, 5}, {3, 4}, {3}}, - {{}} + {{1, 2, 3}, {1, 2}, {1}}, + {{2, 3, 4}, {2, 3}, {2}}, + {{3, 4, 5}, {3, 4}, {3}}, + {{}} }; - JdbcToArrowConfig config = new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), - Calendar.getInstance()) - .setTargetBatchSize(3) - .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) - .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) - .build(); + JdbcToArrowConfig config = + new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()) + .setTargetBatchSize(3) + .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) + .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) + .build(); ArrowVectorIterator iterator = - JdbcToArrow.sqlToArrowVectorIterator(conn.createStatement().executeQuery(table.getQuery()), config); + JdbcToArrow.sqlToArrowVectorIterator( + conn.createStatement().executeQuery(table.getQuery()), config); int batchCount = 0; VectorSchemaRoot prev = null; @@ -178,14 +178,15 @@ public void testVectorSchemaRootReuse() throws SQLException, IOException { @Test public void testJdbcToArrowValuesNoLimit() throws SQLException, IOException { - JdbcToArrowConfig config = new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), - Calendar.getInstance()) - .setTargetBatchSize(JdbcToArrowConfig.NO_LIMIT_BATCH_SIZE) - .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) - .build(); + JdbcToArrowConfig config = + new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()) + .setTargetBatchSize(JdbcToArrowConfig.NO_LIMIT_BATCH_SIZE) + .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) + .build(); ArrowVectorIterator iterator = - JdbcToArrow.sqlToArrowVectorIterator(conn.createStatement().executeQuery(table.getQuery()), config); + JdbcToArrow.sqlToArrowVectorIterator( + conn.createStatement().executeQuery(table.getQuery()), config); validate(iterator); } @@ -195,12 +196,12 @@ public void testTimeStampConsumer() throws SQLException, IOException { final String sql = "select timestamp_field11 from table1"; // first experiment, with calendar and time zone. - JdbcToArrowConfig config = new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), - Calendar.getInstance()) - .setTargetBatchSize(3) - .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) - .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) - .build(); + JdbcToArrowConfig config = + new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()) + .setTargetBatchSize(3) + .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) + .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) + .build(); assertNotNull(config.getCalendar()); try (ArrowVectorIterator iterator = @@ -213,16 +214,16 @@ public void testTimeStampConsumer() throws SQLException, IOException { } // second experiment, without calendar and time zone. - config = new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), - null) - .setTargetBatchSize(3) - .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) - .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) - .build(); + config = + new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), null) + .setTargetBatchSize(3) + .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) + .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP) + .build(); assertNull(config.getCalendar()); try (ArrowVectorIterator iterator = - JdbcToArrow.sqlToArrowVectorIterator(conn.createStatement().executeQuery(sql), config)) { + JdbcToArrow.sqlToArrowVectorIterator(conn.createStatement().executeQuery(sql), config)) { VectorSchemaRoot root = iterator.next(); assertEquals(1, root.getFieldVectors().size()); @@ -278,24 +279,40 @@ private void validate(ArrowVectorIterator iterator) throws SQLException, IOExcep float8Vectors.add((Float8Vector) root.getVector(DOUBLE)); listVectors.add((ListVector) root.getVector(LIST)); } - assertBigIntVectorValues(bigIntVectors, table.getRowCount(), getLongValues(table.getValues(), BIGINT)); - assertTinyIntVectorValues(tinyIntVectors, table.getRowCount(), getIntValues(table.getValues(), TINYINT)); + assertBigIntVectorValues( + bigIntVectors, table.getRowCount(), getLongValues(table.getValues(), BIGINT)); + assertTinyIntVectorValues( + tinyIntVectors, table.getRowCount(), getIntValues(table.getValues(), TINYINT)); assertIntVectorValues(intVectors, table.getRowCount(), getIntValues(table.getValues(), INT)); - assertSmallIntVectorValues(smallIntVectors, table.getRowCount(), getIntValues(table.getValues(), SMALLINT)); - assertBinaryVectorValues(vectorsForBinary, table.getRowCount(), getBinaryValues(table.getValues(), BINARY)); - assertBinaryVectorValues(vectorsForBlob, table.getRowCount(), getBinaryValues(table.getValues(), BLOB)); - assertVarCharVectorValues(vectorsForClob, table.getRowCount(), getCharArray(table.getValues(), CLOB)); - assertVarCharVectorValues(vectorsForVarChar, table.getRowCount(), getCharArray(table.getValues(), VARCHAR)); - assertVarCharVectorValues(vectorsForChar, table.getRowCount(), getCharArray(table.getValues(), CHAR)); + assertSmallIntVectorValues( + smallIntVectors, table.getRowCount(), getIntValues(table.getValues(), SMALLINT)); + assertBinaryVectorValues( + vectorsForBinary, table.getRowCount(), getBinaryValues(table.getValues(), BINARY)); + assertBinaryVectorValues( + vectorsForBlob, table.getRowCount(), getBinaryValues(table.getValues(), BLOB)); + assertVarCharVectorValues( + vectorsForClob, table.getRowCount(), getCharArray(table.getValues(), CLOB)); + assertVarCharVectorValues( + vectorsForVarChar, table.getRowCount(), getCharArray(table.getValues(), VARCHAR)); + assertVarCharVectorValues( + vectorsForChar, table.getRowCount(), getCharArray(table.getValues(), CHAR)); assertBitVectorValues(vectorsForBit, table.getRowCount(), getIntValues(table.getValues(), BIT)); - assertBooleanVectorValues(vectorsForBool, table.getRowCount(), getBooleanValues(table.getValues(), BOOL)); - assertDateDayVectorValues(dateDayVectors, table.getRowCount(), getLongValues(table.getValues(), DATE)); - assertTimeMilliVectorValues(timeMilliVectors, table.getRowCount(), getLongValues(table.getValues(), TIME)); - assertTimeStampVectorValues(timeStampVectors, table.getRowCount(), getLongValues(table.getValues(), TIMESTAMP)); - assertDecimalVectorValues(decimalVectors, table.getRowCount(), getDecimalValues(table.getValues(), DECIMAL)); - assertFloat4VectorValues(float4Vectors, table.getRowCount(), getFloatValues(table.getValues(), REAL)); - assertFloat8VectorValues(float8Vectors, table.getRowCount(), getDoubleValues(table.getValues(), DOUBLE)); - assertListVectorValues(listVectors, table.getRowCount(), getListValues(table.getValues(), LIST)); + assertBooleanVectorValues( + vectorsForBool, table.getRowCount(), getBooleanValues(table.getValues(), BOOL)); + assertDateDayVectorValues( + dateDayVectors, table.getRowCount(), getLongValues(table.getValues(), DATE)); + assertTimeMilliVectorValues( + timeMilliVectors, table.getRowCount(), getLongValues(table.getValues(), TIME)); + assertTimeStampVectorValues( + timeStampVectors, table.getRowCount(), getLongValues(table.getValues(), TIMESTAMP)); + assertDecimalVectorValues( + decimalVectors, table.getRowCount(), getDecimalValues(table.getValues(), DECIMAL)); + assertFloat4VectorValues( + float4Vectors, table.getRowCount(), getFloatValues(table.getValues(), REAL)); + assertFloat8VectorValues( + float8Vectors, table.getRowCount(), getDoubleValues(table.getValues(), DOUBLE)); + assertListVectorValues( + listVectors, table.getRowCount(), getListValues(table.getValues(), LIST)); roots.forEach(root -> root.close()); } @@ -324,7 +341,8 @@ private void assertFloat4VectorValues(List vectors, int rowCount, } } - private void assertDecimalVectorValues(List vectors, int rowCount, BigDecimal[] values) { + private void assertDecimalVectorValues( + List vectors, int rowCount, BigDecimal[] values) { int valueCount = vectors.stream().mapToInt(ValueVector::getValueCount).sum(); assertEquals(rowCount, valueCount); @@ -337,7 +355,8 @@ private void assertDecimalVectorValues(List vectors, int rowCount } } - private void assertTimeStampVectorValues(List vectors, int rowCount, Long[] values) { + private void assertTimeStampVectorValues( + List vectors, int rowCount, Long[] values) { int valueCount = vectors.stream().mapToInt(ValueVector::getValueCount).sum(); assertEquals(rowCount, valueCount); @@ -349,7 +368,8 @@ private void assertTimeStampVectorValues(List vectors, int rowC } } - private void assertTimeMilliVectorValues(List vectors, int rowCount, Long[] values) { + private void assertTimeMilliVectorValues( + List vectors, int rowCount, Long[] values) { int valueCount = vectors.stream().mapToInt(ValueVector::getValueCount).sum(); assertEquals(rowCount, valueCount); @@ -397,7 +417,8 @@ private void assertBooleanVectorValues(List vectors, int rowCount, Bo } } - private void assertVarCharVectorValues(List vectors, int rowCount, byte[][] values) { + private void assertVarCharVectorValues( + List vectors, int rowCount, byte[][] values) { int valueCount = vectors.stream().mapToInt(ValueVector::getValueCount).sum(); assertEquals(rowCount, valueCount); @@ -409,7 +430,8 @@ private void assertVarCharVectorValues(List vectors, int rowCount } } - private void assertBinaryVectorValues(List vectors, int rowCount, byte[][] values) { + private void assertBinaryVectorValues( + List vectors, int rowCount, byte[][] values) { int valueCount = vectors.stream().mapToInt(ValueVector::getValueCount).sum(); assertEquals(rowCount, valueCount); @@ -421,7 +443,8 @@ private void assertBinaryVectorValues(List vectors, int rowCoun } } - private void assertSmallIntVectorValues(List vectors, int rowCount, Integer[] values) { + private void assertSmallIntVectorValues( + List vectors, int rowCount, Integer[] values) { int valueCount = vectors.stream().mapToInt(ValueVector::getValueCount).sum(); assertEquals(rowCount, valueCount); @@ -433,7 +456,8 @@ private void assertSmallIntVectorValues(List vectors, int rowCou } } - private void assertTinyIntVectorValues(List vectors, int rowCount, Integer[] values) { + private void assertTinyIntVectorValues( + List vectors, int rowCount, Integer[] values) { int valueCount = vectors.stream().mapToInt(ValueVector::getValueCount).sum(); assertEquals(rowCount, valueCount); @@ -474,7 +498,8 @@ private void assertIntVectorValues(List vectors, int rowCount, Intege } } - public static void assertListVectorValues(List vectors, int rowCount, Integer[][] values) { + public static void assertListVectorValues( + List vectors, int rowCount, Integer[][] values) { int valueCount = vectors.stream().mapToInt(ValueVector::getValueCount).sum(); assertEquals(rowCount, valueCount); @@ -492,12 +517,11 @@ public static void assertListVectorValues(List vectors, int rowCount } } - /** - * Runs a simple query, and encapsulates the result into a field vector. - */ + /** Runs a simple query, and encapsulates the result into a field vector. */ private FieldVector getQueryResult(JdbcToArrowConfig config) throws SQLException, IOException { - ArrowVectorIterator iterator = JdbcToArrow.sqlToArrowVectorIterator( - conn.createStatement().executeQuery("select real_field8 from table1"), config); + ArrowVectorIterator iterator = + JdbcToArrow.sqlToArrowVectorIterator( + conn.createStatement().executeQuery("select real_field8 from table1"), config); VectorSchemaRoot root = iterator.next(); @@ -513,10 +537,11 @@ private FieldVector getQueryResult(JdbcToArrowConfig config) throws SQLException @Test public void testJdbcToArrowCustomTypeConversion() throws SQLException, IOException { - JdbcToArrowConfigBuilder builder = new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), - Calendar.getInstance()).setTargetBatchSize(JdbcToArrowConfig.NO_LIMIT_BATCH_SIZE) - .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) - .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP); + JdbcToArrowConfigBuilder builder = + new JdbcToArrowConfigBuilder(new RootAllocator(Integer.MAX_VALUE), Calendar.getInstance()) + .setTargetBatchSize(JdbcToArrowConfig.NO_LIMIT_BATCH_SIZE) + .setReuseVectorSchemaRoot(reuseVectorSchemaRoot) + .setArraySubTypeByColumnNameMap(ARRAY_SUB_TYPE_BY_COLUMN_NAME_MAP); // first experiment, using default type converter JdbcToArrowConfig config = builder.build(); @@ -527,15 +552,16 @@ public void testJdbcToArrowCustomTypeConversion() throws SQLException, IOExcepti } // second experiment, using customized type converter - builder.setJdbcToArrowTypeConverter((fieldInfo) -> { - switch (fieldInfo.getJdbcType()) { - case Types.REAL: - // this is different from the default type converter - return new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE); - default: - return null; - } - }); + builder.setJdbcToArrowTypeConverter( + (fieldInfo) -> { + switch (fieldInfo.getJdbcType()) { + case Types.REAL: + // this is different from the default type converter + return new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE); + default: + return null; + } + }); config = builder.build(); try (FieldVector vector = getQueryResult(config)) { diff --git a/java/adapter/orc/pom.xml b/java/adapter/orc/pom.xml index bc89c4698eecf..7df08e1a98b36 100644 --- a/java/adapter/orc/pom.xml +++ b/java/adapter/orc/pom.xml @@ -24,9 +24,13 @@ jar Arrow Orc Adapter (Experimental/Contrib)A JNI wrapper for the C++ ORC reader implementation. + ../../../cpp/release-build/ + dev/checkstyle/checkstyle-spotless.xml + none + org.apache.arrow diff --git a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcFieldNode.java b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcFieldNode.java index 716a13876608c..faf48e19445ae 100644 --- a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcFieldNode.java +++ b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcFieldNode.java @@ -14,12 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.orc; -/** - * Metadata about Vectors/Arrays that is passed via JNI interface. - */ +/** Metadata about Vectors/Arrays that is passed via JNI interface. */ class OrcFieldNode { private final int length; @@ -27,6 +24,7 @@ class OrcFieldNode { /** * Construct a new instance. + * * @param length the number of values written. * @param nullCount the number of null values. */ diff --git a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcJniUtils.java b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcJniUtils.java index d61799e990f77..692b0c061839c 100644 --- a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcJniUtils.java +++ b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcJniUtils.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.orc; import java.io.File; @@ -25,24 +24,21 @@ import java.nio.file.StandardCopyOption; import java.util.Locale; -/** - * Helper class for JNI related operations. - */ +/** Helper class for JNI related operations. */ class OrcJniUtils { private static final String LIBRARY_NAME = "arrow_orc_jni"; private static boolean isLoaded = false; - private OrcJniUtils() { - } + private OrcJniUtils() {} - static void loadOrcAdapterLibraryFromJar() - throws IOException, IllegalAccessException { + static void loadOrcAdapterLibraryFromJar() throws IOException, IllegalAccessException { synchronized (OrcJniUtils.class) { if (!isLoaded) { final String libraryToLoad = LIBRARY_NAME + "/" + getNormalizedArch() + "/" + System.mapLibraryName(LIBRARY_NAME); final File libraryFile = - moveFileFromJarToTemp(System.getProperty("java.io.tmpdir"), libraryToLoad, LIBRARY_NAME); + moveFileFromJarToTemp( + System.getProperty("java.io.tmpdir"), libraryToLoad, LIBRARY_NAME); System.load(libraryFile.getAbsolutePath()); isLoaded = true; } @@ -64,11 +60,11 @@ private static String getNormalizedArch() { return arch; } - private static File moveFileFromJarToTemp(final String tmpDir, String libraryToLoad, String libraryName) - throws IOException { + private static File moveFileFromJarToTemp( + final String tmpDir, String libraryToLoad, String libraryName) throws IOException { final File temp = File.createTempFile(tmpDir, libraryName); - try (final InputStream is = OrcReaderJniWrapper.class.getClassLoader() - .getResourceAsStream(libraryToLoad)) { + try (final InputStream is = + OrcReaderJniWrapper.class.getClassLoader().getResourceAsStream(libraryToLoad)) { if (is == null) { throw new FileNotFoundException(libraryToLoad); } else { diff --git a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcMemoryJniWrapper.java b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcMemoryJniWrapper.java index 473e8314243b1..70f2a655654c6 100644 --- a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcMemoryJniWrapper.java +++ b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcMemoryJniWrapper.java @@ -14,12 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.orc; -/** - * Wrapper for orc memory allocated by native code. - */ +/** Wrapper for orc memory allocated by native code. */ class OrcMemoryJniWrapper implements AutoCloseable { private final long nativeInstanceId; @@ -32,6 +29,7 @@ class OrcMemoryJniWrapper implements AutoCloseable { /** * Construct a new instance. + * * @param nativeInstanceId unique id of the underlying memory. * @param memoryAddress starting memory address of the underlying memory. * @param size size of the valid data. @@ -46,6 +44,7 @@ class OrcMemoryJniWrapper implements AutoCloseable { /** * Return the size of underlying chunk of memory that has valid data. + * * @return valid data size */ long getSize() { @@ -54,6 +53,7 @@ long getSize() { /** * Return the size of underlying chunk of memory managed by this OrcMemoryJniWrapper. + * * @return underlying memory size */ long getCapacity() { @@ -62,6 +62,7 @@ long getCapacity() { /** * Return the memory address of underlying chunk of memory. + * * @return memory address */ long getMemoryAddress() { diff --git a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcReader.java b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcReader.java index 648e17e9c374c..ca9b44e7e8123 100644 --- a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcReader.java +++ b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcReader.java @@ -14,44 +14,42 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.orc; import java.io.IOException; - import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.ipc.ArrowReader; /** - * Orc Reader that allow accessing orc stripes in Orc file. - * This orc reader basically acts like an ArrowReader iterator that - * iterate over orc stripes. Each stripe will be accessed via an - * ArrowReader. + * Orc Reader that allow accessing orc stripes in Orc file. This orc reader basically acts like an + * ArrowReader iterator that iterate over orc stripes. Each stripe will be accessed via an + * ArrowReader. */ public class OrcReader implements AutoCloseable { private final OrcReaderJniWrapper jniWrapper; private BufferAllocator allocator; - /** - * reference to native reader instance. - */ + /** reference to native reader instance. */ private final long nativeInstanceId; /** * Create an OrcReader that iterate over orc stripes. + * * @param filePath file path to target file, currently only support local file. * @param allocator allocator provided to ArrowReader. * @throws IOException throws exception in case of file not found */ - public OrcReader(String filePath, BufferAllocator allocator) throws IOException, IllegalAccessException { + public OrcReader(String filePath, BufferAllocator allocator) + throws IOException, IllegalAccessException { this.allocator = allocator; this.jniWrapper = OrcReaderJniWrapper.getInstance(); this.nativeInstanceId = jniWrapper.open(filePath); } /** - * Seek to designated row. Invoke NextStripeReader() after seek - * will return stripe reader starting from designated row. + * Seek to designated row. Invoke NextStripeReader() after seek will return stripe reader starting + * from designated row. + * * @param rowNumber the rows number to seek * @return true if seek operation is succeeded */ diff --git a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcReaderJniWrapper.java b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcReaderJniWrapper.java index ff449c343c4e7..be57485005fbf 100644 --- a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcReaderJniWrapper.java +++ b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcReaderJniWrapper.java @@ -14,14 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.orc; import java.io.IOException; -/** - * JNI wrapper for Orc reader. - */ +/** JNI wrapper for Orc reader. */ class OrcReaderJniWrapper { private static volatile OrcReaderJniWrapper INSTANCE; @@ -41,21 +38,24 @@ static OrcReaderJniWrapper getInstance() throws IOException, IllegalAccessExcept /** * Construct a orc file reader over the target file. + * * @param fileName absolute file path of target file - * @return id of the orc reader instance if file opened successfully, - * otherwise return error code * -1. + * @return id of the orc reader instance if file opened successfully, otherwise return error code + * * -1. */ native long open(String fileName); /** * Release resources associated with designated reader instance. + * * @param readerId id of the reader instance. */ native void close(long readerId); /** - * Seek to designated row. Invoke nextStripeReader() after seek - * will return id of stripe reader starting from designated row. + * Seek to designated row. Invoke nextStripeReader() after seek will return id of stripe reader + * starting from designated row. + * * @param readerId id of the reader instance * @param rowNumber the rows number to seek * @return true if seek operation is succeeded @@ -64,6 +64,7 @@ static OrcReaderJniWrapper getInstance() throws IOException, IllegalAccessExcept /** * The number of stripes in the file. + * * @param readerId id of the reader instance * @return number of stripes */ @@ -71,6 +72,7 @@ static OrcReaderJniWrapper getInstance() throws IOException, IllegalAccessExcept /** * Get a stripe level ArrowReader with specified batchSize in each record batch. + * * @param readerId id of the reader instance * @param batchSize the number of rows loaded on each iteration * @return id of the stripe reader instance. diff --git a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcRecordBatch.java b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcRecordBatch.java index a006cacab98f2..f78898df2205d 100644 --- a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcRecordBatch.java +++ b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcRecordBatch.java @@ -14,27 +14,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.orc; import java.util.Arrays; import java.util.List; -/** - * Wrapper for record batch meta and native memory. - */ +/** Wrapper for record batch meta and native memory. */ class OrcRecordBatch { final int length; - /** - * Nodes correspond to the pre-ordered flattened logical schema. - */ + /** Nodes correspond to the pre-ordered flattened logical schema. */ final List nodes; final List buffers; /** * Construct a new instance. + * * @param length number of records included in current batch * @param nodes meta data for each fields * @param buffers buffers for underlying data diff --git a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcReferenceManager.java b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcReferenceManager.java index fdec337e85d39..38233a0493bef 100644 --- a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcReferenceManager.java +++ b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcReferenceManager.java @@ -14,11 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.orc; import java.util.concurrent.atomic.AtomicInteger; - import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.OwnershipTransferResult; @@ -26,8 +24,8 @@ import org.apache.arrow.util.Preconditions; /** - * A simple reference manager implementation for memory allocated by native code. - * The underlying memory will be released when reference count reach zero. + * A simple reference manager implementation for memory allocated by native code. The underlying + * memory will be released when reference count reach zero. */ public class OrcReferenceManager implements ReferenceManager { private final AtomicInteger bufRefCnt = new AtomicInteger(0); @@ -50,8 +48,8 @@ public boolean release() { @Override public boolean release(int decrement) { - Preconditions.checkState(decrement >= 1, - "ref count decrement should be greater than or equal to 1"); + Preconditions.checkState( + decrement >= 1, "ref count decrement should be greater than or equal to 1"); // decrement the ref count final int refCnt; synchronized (this) { @@ -89,18 +87,21 @@ public ArrowBuf deriveBuffer(ArrowBuf sourceBuffer, long index, long length) { final long derivedBufferAddress = sourceBuffer.memoryAddress() + index; // create new ArrowBuf - final ArrowBuf derivedBuf = new ArrowBuf( + final ArrowBuf derivedBuf = + new ArrowBuf( this, null, length, // length (in bytes) in the underlying memory chunk for this new ArrowBuf - derivedBufferAddress // starting byte address in the underlying memory for this new ArrowBuf, + derivedBufferAddress // starting byte address in the underlying memory for this new + // ArrowBuf, ); return derivedBuf; } @Override - public OwnershipTransferResult transferOwnership(ArrowBuf sourceBuffer, BufferAllocator targetAllocator) { + public OwnershipTransferResult transferOwnership( + ArrowBuf sourceBuffer, BufferAllocator targetAllocator) { throw new UnsupportedOperationException(); } diff --git a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcStripeReader.java b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcStripeReader.java index 484296d92e039..52f5cf429a48d 100644 --- a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcStripeReader.java +++ b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcStripeReader.java @@ -14,13 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.orc; import java.io.IOException; import java.util.ArrayList; import java.util.stream.Collectors; - import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.ipc.ArrowReader; @@ -33,19 +31,16 @@ import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel; -/** - * Orc stripe that load data into ArrowRecordBatch. - */ +/** Orc stripe that load data into ArrowRecordBatch. */ public class OrcStripeReader extends ArrowReader { - /** - * reference to native stripe reader instance. - */ + /** reference to native stripe reader instance. */ private final long nativeInstanceId; /** * Construct a new instance. - * @param nativeInstanceId nativeInstanceId of the stripe reader instance, obtained by - * calling nextStripeReader from OrcReaderJniWrapper + * + * @param nativeInstanceId nativeInstanceId of the stripe reader instance, obtained by calling + * nextStripeReader from OrcReaderJniWrapper * @param allocator memory allocator for accounting. */ OrcStripeReader(long nativeInstanceId, BufferAllocator allocator) { @@ -62,18 +57,20 @@ public boolean loadNextBatch() throws IOException { ArrayList buffers = new ArrayList<>(); for (OrcMemoryJniWrapper buffer : recordBatch.buffers) { - buffers.add(new ArrowBuf( + buffers.add( + new ArrowBuf( new OrcReferenceManager(buffer), null, (int) buffer.getSize(), buffer.getMemoryAddress())); } - loadRecordBatch(new ArrowRecordBatch( + loadRecordBatch( + new ArrowRecordBatch( recordBatch.length, recordBatch.nodes.stream() - .map(buf -> new ArrowFieldNode(buf.getLength(), buf.getNullCount())) - .collect(Collectors.toList()), + .map(buf -> new ArrowFieldNode(buf.getLength(), buf.getNullCount())) + .collect(Collectors.toList()), buffers)); return true; } @@ -83,7 +80,6 @@ public long bytesRead() { return 0; } - @Override protected void closeReadSource() throws IOException { OrcStripeReaderJniWrapper.close(nativeInstanceId); @@ -94,9 +90,8 @@ protected Schema readSchema() throws IOException { byte[] schemaBytes = OrcStripeReaderJniWrapper.getSchema(nativeInstanceId); try (MessageChannelReader schemaReader = - new MessageChannelReader( - new ReadChannel( - new ByteArrayReadableSeekableByteChannel(schemaBytes)), allocator)) { + new MessageChannelReader( + new ReadChannel(new ByteArrayReadableSeekableByteChannel(schemaBytes)), allocator)) { MessageResult result = schemaReader.readNext(); if (result == null) { diff --git a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcStripeReaderJniWrapper.java b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcStripeReaderJniWrapper.java index 1dd96986108b4..e7b691087fb96 100644 --- a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcStripeReaderJniWrapper.java +++ b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcStripeReaderJniWrapper.java @@ -14,16 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.orc; -/** - * JNI wrapper for orc stripe reader. - */ +/** JNI wrapper for orc stripe reader. */ class OrcStripeReaderJniWrapper { /** * Get the schema of current stripe. + * * @param readerId id of the stripe reader instance. * @return serialized schema. */ @@ -31,14 +29,15 @@ class OrcStripeReaderJniWrapper { /** * Load next record batch. + * * @param readerId id of the stripe reader instance. - * @return loaded record batch, return null when reached - * the end of current stripe. + * @return loaded record batch, return null when reached the end of current stripe. */ static native OrcRecordBatch next(long readerId); /** * Release resources of underlying reader. + * * @param readerId id of the stripe reader instance. */ static native void close(long readerId); diff --git a/java/adapter/orc/src/test/java/org/apache/arrow/adapter/orc/OrcReaderTest.java b/java/adapter/orc/src/test/java/org/apache/arrow/adapter/orc/OrcReaderTest.java index 4153a35a61c67..17098806be72a 100644 --- a/java/adapter/orc/src/test/java/org/apache/arrow/adapter/orc/OrcReaderTest.java +++ b/java/adapter/orc/src/test/java/org/apache/arrow/adapter/orc/OrcReaderTest.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.arrow.adapter.orc; import static org.junit.Assert.assertEquals; @@ -24,8 +23,6 @@ import java.io.File; import java.nio.charset.StandardCharsets; import java.util.List; - - import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.IntVector; @@ -45,11 +42,9 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; - public class OrcReaderTest { - @Rule - public TemporaryFolder testFolder = new TemporaryFolder(); + @Rule public TemporaryFolder testFolder = new TemporaryFolder(); private static final int MAX_ALLOCATION = 8 * 1024; private static RootAllocator allocator; @@ -64,8 +59,10 @@ public void testOrcJniReader() throws Exception { TypeDescription schema = TypeDescription.fromString("struct"); File testFile = new File(testFolder.getRoot(), "test-orc"); - Writer writer = OrcFile.createWriter(new Path(testFile.getAbsolutePath()), - OrcFile.writerOptions(new Configuration()).setSchema(schema)); + Writer writer = + OrcFile.createWriter( + new Path(testFile.getAbsolutePath()), + OrcFile.writerOptions(new Configuration()).setSchema(schema)); VectorizedRowBatch batch = schema.createRowBatch(); LongColumnVector longColumnVector = (LongColumnVector) batch.cols[0]; BytesColumnVector bytesColumnVector = (BytesColumnVector) batch.cols[1]; diff --git a/java/dev/checkstyle/checkstyle-spotless.xml b/java/dev/checkstyle/checkstyle-spotless.xml index cbaec1a39bf2c..a2e9a60b12c72 100644 --- a/java/dev/checkstyle/checkstyle-spotless.xml +++ b/java/dev/checkstyle/checkstyle-spotless.xml @@ -89,7 +89,7 @@ - --> + @@ -99,7 +99,7 @@ value="WhitespaceAround: ''{0}'' is not followed by whitespace. Empty blocks may only be represented as '{}' when not part of a multi-block statement (4.1.3)"/> - + -->