Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

GH-37069: [C#] Allow for record batches larger than 2 GB #38593

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
+5 −6
Draft

GH-37069: [C#] Allow for record batches larger than 2 GB #38593

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion csharp/src/Apache.Arrow.Flight/Internal/FlightDataStream.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ public async Task Write(RecordBatch recordBatch, ByteString applicationMetadata)
await _clientStreamWriter.WriteAsync(_currentFlightData).ConfigureAwait(false);
}

private protected override ValueTask<long> WriteMessageAsync<T>(MessageHeader headerType, Offset<T> headerOffset, int bodyLength, CancellationToken cancellationToken)
private protected override ValueTask<long> WriteMessageAsync<T>(MessageHeader headerType, Offset<T> headerOffset, long bodyLength, CancellationToken cancellationToken)
{
Offset<Flatbuf.Message> messageOffset = Flatbuf.Message.CreateMessage(
Builder, CurrentMetadataVersion, headerType, headerOffset.Value,
Expand Down
1 change: 0 additions & 1 deletion csharp/src/Apache.Arrow/ArrowBuffer.Builder.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,5 @@ private void Reallocate(int numBytes)
}

}

}
}
8 changes: 4 additions & 4 deletions csharp/src/Apache.Arrow/Ipc/ArrowStreamWriter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ public Buffer(ArrowBuffer buffer, int offset)

public IReadOnlyList<Buffer> Buffers => _buffers;

public int TotalLength { get; private set; }
public long TotalLength { get; private set; }

public ArrowRecordBatchFlatBufferBuilder()
{
Expand Down Expand Up @@ -204,7 +204,7 @@ private void CreateBuffers<T>(PrimitiveArray<T> array)

private Buffer CreateBuffer(ArrowBuffer buffer)
{
int offset = TotalLength;
int offset = checked((int)TotalLength);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Both "offset" and the properties in the Buffer class would also need to be changed to long.


int paddedLength = checked((int)BitUtility.RoundUpToMultipleOf8(buffer.Length));
TotalLength += paddedLength;
Expand Down Expand Up @@ -823,7 +823,7 @@ await WriteMessageAsync(Flatbuf.MessageHeader.Schema, schemaOffset, 0, cancellat
/// The number of bytes written to the stream.
/// </returns>
private protected long WriteMessage<T>(
Flatbuf.MessageHeader headerType, Offset<T> headerOffset, int bodyLength)
Flatbuf.MessageHeader headerType, Offset<T> headerOffset, long bodyLength)
where T : struct
{
Offset<Flatbuf.Message> messageOffset = Flatbuf.Message.CreateMessage(
Expand Down Expand Up @@ -853,7 +853,7 @@ private protected long WriteMessage<T>(
/// The number of bytes written to the stream.
/// </returns>
private protected virtual async ValueTask<long> WriteMessageAsync<T>(
Flatbuf.MessageHeader headerType, Offset<T> headerOffset, int bodyLength,
Flatbuf.MessageHeader headerType, Offset<T> headerOffset, long bodyLength,
CancellationToken cancellationToken)
where T : struct
{
Expand Down
Loading