Skip to content

Commit

Permalink
fix: rebase
Browse files Browse the repository at this point in the history
  • Loading branch information
@vibhatha
vibhatha committed Jul 2, 2024
1 parent 24c10dc commit 7445a6d
Show file tree
Hide file tree
Showing 4 changed files with 112 additions and 59 deletions.
34 changes: 21 additions & 13 deletions java/vector/src/main/java/org/apache/arrow/vector/complex/ListViewVector.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -486,8 +486,12 @@ public void transfer() {

@Override
public void splitAndTransfer(int startIndex, int length) {
Preconditions.checkArgument(startIndex >= 0 && length >= 0 && startIndex + length <= valueCount,
"Invalid parameters startIndex: %s, length: %s for valueCount: %s", startIndex, length, valueCount);
Preconditions.checkArgument(
startIndex >= 0 && length >= 0 && startIndex + length <= valueCount,
"Invalid parameters startIndex: %s, length: %s for valueCount: %s",
startIndex,
length,
valueCount);
to.clear();
if (length > 0) {
final int startPoint = offsetBuffer.getInt((long) startIndex * OFFSET_WIDTH);
Expand Down Expand Up @@ -554,8 +558,11 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, ListView
target.allocateValidityBuffer(byteSizeTarget);

for (int i = 0; i < byteSizeTarget - 1; i++) {
byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset);
byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, firstByteSource + i + 1, offset);
byte b1 =
BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset);
byte b2 =
BitVectorHelper.getBitsFromNextByte(
validityBuffer, firstByteSource + i + 1, offset);

target.validityBuffer.setByte(i, (b1 + b2));
}
Expand All @@ -570,15 +577,18 @@ private void splitAndTransferValidityBuffer(int startIndex, int length, ListView
* by shifting data from the current byte.
*/
if ((firstByteSource + byteSizeTarget - 1) < lastByteSource) {
byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer,
firstByteSource + byteSizeTarget - 1, offset);
byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer,
firstByteSource + byteSizeTarget, offset);
byte b1 =
BitVectorHelper.getBitsFromCurrentByte(
validityBuffer, firstByteSource + byteSizeTarget - 1, offset);
byte b2 =
BitVectorHelper.getBitsFromNextByte(
validityBuffer, firstByteSource + byteSizeTarget, offset);

target.validityBuffer.setByte(byteSizeTarget - 1, b1 + b2);
} else {
byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer,
firstByteSource + byteSizeTarget - 1, offset);
byte b1 =
BitVectorHelper.getBitsFromCurrentByte(
validityBuffer, firstByteSource + byteSizeTarget - 1, offset);
target.validityBuffer.setByte(byteSizeTarget - 1, b1);
}
}
Expand All @@ -591,9 +601,7 @@ public ValueVector getTo() {
}

@Override
public void copyValueSafe(int from, int to) {

}
public void copyValueSafe(int from, int to) {}
}

@Override
Expand Down
12 changes: 6 additions & 6 deletions java/vector/src/main/java/org/apache/arrow/vector/complex/impl/UnionListViewReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.arrow.vector.complex.impl;

import org.apache.arrow.vector.ValueVector;
Expand All @@ -25,9 +24,7 @@
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.Field;

/**
* {@link FieldReader} for listview of union types.
*/
/** {@link FieldReader} for listview of union types. */
public class UnionListViewReader extends AbstractFieldReader {

private final ListViewVector vector;
Expand All @@ -37,6 +34,7 @@ public class UnionListViewReader extends AbstractFieldReader {

/**
* Constructor for UnionListViewReader.
*
* @param vector the vector to read from
*/
public UnionListViewReader(ListViewVector vector) {
Expand All @@ -61,7 +59,8 @@ public void setPosition(int index) {
currentOffset = 0;
size = 0;
} else {
currentOffset = vector.getOffsetBuffer().getInt(index * (long) BaseRepeatedValueViewVector.OFFSET_WIDTH);
currentOffset =
vector.getOffsetBuffer().getInt(index * (long) BaseRepeatedValueViewVector.OFFSET_WIDTH);
size = vector.getSizeBuffer().getInt(index * (long) BaseRepeatedValueViewVector.SIZE_WIDTH);
}
}
Expand Down Expand Up @@ -98,7 +97,8 @@ public int size() {

@Override
public boolean next() {
// Here, the currentOffSet keeps track of the current position in the vector inside the list at set position.
// Here, the currentOffSet keeps track of the current position in the vector inside the list at
// set position.
// And, size keeps track of the elements count in the list, so to make sure we traverse
// the full list, we need to check if the currentOffset is less than the currentOffset + size
if (currentOffset < currentOffset + size) {
Expand Down
103 changes: 71 additions & 32 deletions java/vector/src/test/java/org/apache/arrow/vector/TestListViewVector.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1656,8 +1656,12 @@ public void testOutOfOrderOffset1() {
}
}

private int validateSizeBufferAndCalculateMinOffset(int start, int splitLength, ArrowBuf fromOffsetBuffer,
ArrowBuf fromSizeBuffer, ArrowBuf toSizeBuffer) {
private int validateSizeBufferAndCalculateMinOffset(
int start,
int splitLength,
ArrowBuf fromOffsetBuffer,
ArrowBuf fromSizeBuffer,
ArrowBuf toSizeBuffer) {
int minOffset = fromOffsetBuffer.getInt((long) start * ListViewVector.OFFSET_WIDTH);
int fromDataLength;
int toDataLength;
Expand All @@ -1667,7 +1671,9 @@ private int validateSizeBufferAndCalculateMinOffset(int start, int splitLength,
toDataLength = toSizeBuffer.getInt((long) (i) * ListViewVector.SIZE_WIDTH);

/* validate size */
assertEquals(fromDataLength, toDataLength,
assertEquals(
fromDataLength,
toDataLength,
"Different data lengths at index: " + i + " and start: " + start);

/* calculate minimum offset */
Expand All @@ -1680,46 +1686,65 @@ private int validateSizeBufferAndCalculateMinOffset(int start, int splitLength,
return minOffset;
}

private void validateOffsetBuffer(int start, int splitLength, ArrowBuf fromOffsetBuffer,
ArrowBuf toOffsetBuffer, int minOffset) {
private void validateOffsetBuffer(
int start,
int splitLength,
ArrowBuf fromOffsetBuffer,
ArrowBuf toOffsetBuffer,
int minOffset) {
int offset1;
int offset2;

for (int i = 0; i < splitLength; i++) {
offset1 = fromOffsetBuffer.getInt((long) (start + i) * ListViewVector.OFFSET_WIDTH);
offset2 = toOffsetBuffer.getInt((long) (i) * ListViewVector.OFFSET_WIDTH);
assertEquals(offset1 - minOffset, offset2,
assertEquals(
offset1 - minOffset,
offset2,
"Different offset values at index: " + i + " and start: " + start);
}
}

private void validateDataBuffer(int start, int splitLength, ArrowBuf fromOffsetBuffer, ArrowBuf fromSizeBuffer,
BigIntVector fromDataVector, ArrowBuf toOffsetBuffer, BigIntVector toDataVector) {
private void validateDataBuffer(
int start,
int splitLength,
ArrowBuf fromOffsetBuffer,
ArrowBuf fromSizeBuffer,
BigIntVector fromDataVector,
ArrowBuf toOffsetBuffer,
BigIntVector toDataVector) {
int dataLength;
Long fromValue;
for (int i = 0; i < splitLength; i++) {
dataLength = fromSizeBuffer.getInt((long) (start + i) * ListViewVector.SIZE_WIDTH);
for (int j = 0; j < dataLength; j++) {
fromValue = fromDataVector.getObject(
(fromOffsetBuffer.getInt((long) (start + i) * ListViewVector.OFFSET_WIDTH) + j));
Long toValue = toDataVector.getObject(
(toOffsetBuffer.getInt((long) i * ListViewVector.OFFSET_WIDTH) + j));
assertEquals(fromValue, toValue,
"Different data values at index: " + i + " and start: " + start);
fromValue =
fromDataVector.getObject(
(fromOffsetBuffer.getInt((long) (start + i) * ListViewVector.OFFSET_WIDTH) + j));
Long toValue =
toDataVector.getObject(
(toOffsetBuffer.getInt((long) i * ListViewVector.OFFSET_WIDTH) + j));
assertEquals(
fromValue, toValue, "Different data values at index: " + i + " and start: " + start);
}
}
}

/**
* Validate split and transfer of data from fromVector to toVector.
* Note that this method assumes that the child vector is BigIntVector.
* Validate split and transfer of data from fromVector to toVector. Note that this method assumes
* that the child vector is BigIntVector.
*
* @param start start index
* @param splitLength length of data to split and transfer
* @param fromVector fromVector
* @param toVector toVector
*/
private void validateSplitAndTransfer(TransferPair transferPair, int start, int splitLength,
ListViewVector fromVector, ListViewVector toVector) {
private void validateSplitAndTransfer(
TransferPair transferPair,
int start,
int splitLength,
ListViewVector fromVector,
ListViewVector toVector) {

transferPair.splitAndTransfer(start, splitLength);

Expand All @@ -1742,14 +1767,20 @@ private void validateSplitAndTransfer(TransferPair transferPair, int start, int
BigIntVector fromDataVector = (BigIntVector) fromVector.getDataVector();

/* validate size buffers */
int minOffset = validateSizeBufferAndCalculateMinOffset(start, splitLength,
fromOffsetBuffer, fromSizeBuffer, toSizeBuffer);
int minOffset =
validateSizeBufferAndCalculateMinOffset(
start, splitLength, fromOffsetBuffer, fromSizeBuffer, toSizeBuffer);
/* validate offset buffers */
validateOffsetBuffer(start, splitLength, fromOffsetBuffer,
toOffsetBuffer, minOffset);
validateOffsetBuffer(start, splitLength, fromOffsetBuffer, toOffsetBuffer, minOffset);
/* validate data */
validateDataBuffer(start, splitLength, fromOffsetBuffer, fromSizeBuffer,
fromDataVector, toOffsetBuffer, toDataVector);
validateDataBuffer(
start,
splitLength,
fromOffsetBuffer,
fromSizeBuffer,
fromDataVector,
toOffsetBuffer,
toDataVector);
}

@Test
Expand Down Expand Up @@ -1831,7 +1862,9 @@ public void testSplitAndTransfer() throws Exception {
offset++;
actual = dataVector.getObject(offset);
assertEquals(Long.valueOf(12), actual);
assertEquals(Integer.toString(3), Integer.toString(sizeBuffer.getInt(index * ListViewVector.SIZE_WIDTH)));
assertEquals(
Integer.toString(3),
Integer.toString(sizeBuffer.getInt(index * ListViewVector.SIZE_WIDTH)));

/* index 1 */
index++;
Expand All @@ -1846,7 +1879,9 @@ public void testSplitAndTransfer() throws Exception {
actual = dataVector.getObject(offset);
assertEquals(Long.valueOf(14), actual);
size++;
assertEquals(Integer.toString(size), Integer.toString(sizeBuffer.getInt(index * ListViewVector.SIZE_WIDTH)));
assertEquals(
Integer.toString(size),
Integer.toString(sizeBuffer.getInt(index * ListViewVector.SIZE_WIDTH)));

/* index 2 */
size = 0;
Expand All @@ -1870,8 +1905,9 @@ public void testSplitAndTransfer() throws Exception {
size++;
actual = dataVector.getObject(offset);
assertEquals(Long.valueOf(18), actual);
assertEquals(Integer.toString(size), Integer.toString(sizeBuffer.getInt(index * ListViewVector.SIZE_WIDTH)));

assertEquals(
Integer.toString(size),
Integer.toString(sizeBuffer.getInt(index * ListViewVector.SIZE_WIDTH)));

/* index 3 */
size = 0;
Expand All @@ -1883,7 +1919,9 @@ public void testSplitAndTransfer() throws Exception {
actual = dataVector.getObject(offset);
assertEquals(Long.valueOf(19), actual);
size++;
assertEquals(Integer.toString(size), Integer.toString(sizeBuffer.getInt(index * ListViewVector.SIZE_WIDTH)));
assertEquals(
Integer.toString(size),
Integer.toString(sizeBuffer.getInt(index * ListViewVector.SIZE_WIDTH)));

/* index 4 */
size = 0;
Expand All @@ -1907,7 +1945,9 @@ public void testSplitAndTransfer() throws Exception {
actual = dataVector.getObject(offset);
assertEquals(Long.valueOf(23), actual);
size++;
assertEquals(Integer.toString(size), Integer.toString(sizeBuffer.getInt(index * ListViewVector.SIZE_WIDTH)));
assertEquals(
Integer.toString(size),
Integer.toString(sizeBuffer.getInt(index * ListViewVector.SIZE_WIDTH)));

/* do split and transfer */
try (ListViewVector toVector = ListViewVector.empty("toVector", allocator)) {
Expand All @@ -1932,8 +1972,7 @@ public void testOutOfOrderOffsetSplitAndTransfer() {

// Initialize the child vector using `initializeChildrenFromFields` method.

FieldType fieldType = new FieldType(true, new ArrowType.Int(64, true),
null, null);
FieldType fieldType = new FieldType(true, new ArrowType.Int(64, true), null, null);
Field field = new Field("child-vector", fieldType, null);
fromVector.initializeChildrenFromFields(Collections.singletonList(field));

Expand Down
22 changes: 14 additions & 8 deletions java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,13 +204,15 @@ private void createListTypeVectorWithDecimalType(FieldWriter writer, DecimalHold
if (j % 4 == 0) {
writer.writeDecimal(new BigDecimal(j));
} else if (j % 4 == 1) {
DecimalUtility.writeBigDecimalToArrowBuf(new BigDecimal(j), holder.buffer, 0, DecimalVector.TYPE_WIDTH);
DecimalUtility.writeBigDecimalToArrowBuf(
new BigDecimal(j), holder.buffer, 0, DecimalVector.TYPE_WIDTH);
holder.start = 0;
holder.scale = 0;
holder.precision = 10;
writer.write(holder);
} else if (j % 4 == 2) {
DecimalUtility.writeBigDecimalToArrowBuf(new BigDecimal(j), holder.buffer, 0, DecimalVector.TYPE_WIDTH);
DecimalUtility.writeBigDecimalToArrowBuf(
new BigDecimal(j), holder.buffer, 0, DecimalVector.TYPE_WIDTH);
writer.writeDecimal(0, holder.buffer, arrowType);
} else {
byte[] value = BigDecimal.valueOf(j).unscaledValue().toByteArray();
Expand Down Expand Up @@ -456,7 +458,8 @@ private void checkScalarTypeVectorWithNullableType(FieldReader reader) {
}
}

private void createListTypeVectorWithStructType(FieldWriter fieldWriter, StructWriter structWriter) {
private void createListTypeVectorWithStructType(
FieldWriter fieldWriter, StructWriter structWriter) {
for (int i = 0; i < COUNT; i++) {
fieldWriter.startList();
for (int j = 0; j < i % 7; j++) {
Expand Down Expand Up @@ -621,7 +624,8 @@ private void checkListTypeMap(FieldReader reader) {

/* Test Cases */

private void createListTypeVectorWithFixedSizeBinaryType(FieldWriter writer, List<ArrowBuf> buffers) {
private void createListTypeVectorWithFixedSizeBinaryType(
FieldWriter writer, List<ArrowBuf> buffers) {
for (int i = 0; i < COUNT; i++) {
writer.startList();
for (int j = 0; j < i % 7; j++) {
Expand Down Expand Up @@ -897,7 +901,8 @@ public void listViewListType() {
}

/**
* This test is similar to {@link #listListType()} but we get the inner list writer once at the beginning.
* This test is similar to {@link #listListType()} but we get the inner list writer once at the
* beginning.
*/
@Test
public void listListType2() {
Expand Down Expand Up @@ -964,8 +969,8 @@ public void unionListListType2() {
}

/**
* This test is similar to {@link #unionListViewListType()}
* but we get the inner list writer once at the beginning.
* This test is similar to {@link #unionListViewListType()} but we get the inner list writer once
* at the beginning.
*/
@Test
public void unionListViewListType2() {
Expand Down Expand Up @@ -1598,7 +1603,8 @@ public void testSingleStructWriter1() {
Float4Reader float4Reader = singleStructReader.reader("float4Field");
Float8Reader float8Reader = singleStructReader.reader("float8Field");
UnionListReader listReader = (UnionListReader) singleStructReader.reader("listField");
// UnionListViewReader listViewReader = (UnionListViewReader) singleStructReader.reader("listViewField");
// UnionListViewReader listViewReader = (UnionListViewReader)
// singleStructReader.reader("listViewField");
UnionMapReader mapReader = (UnionMapReader) singleStructReader.reader("mapField");

for (int i = 0; i < initialCapacity; i++) {
Expand Down

0 comments on commit 7445a6d

Please sign in to comment.