Skip to content

Commit

Permalink
Avoid copy/allocation when read view types from parquet (#5877)
Browse files Browse the repository at this point in the history
* avoid copy/allocation when build from offset buffer

* avoid hard code block id
  • Loading branch information
XiangpengHao authored Jun 13, 2024
1 parent 2d17bf0 commit c6359bf
Showing 1 changed file with 8 additions and 5 deletions.
13 changes: 8 additions & 5 deletions parquet/src/arrow/buffer/offset_buffer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -156,16 +156,19 @@ impl<I: OffsetSizeTrait> OffsetBuffer<I> {

fn build_generic_byte_view(self) -> GenericByteViewBuilder<BinaryViewType> {
let mut builder = GenericByteViewBuilder::<BinaryViewType>::with_capacity(self.len());
let mut values = self.values;
let buffer = self.values.into();
let block = builder.append_block(buffer);
for window in self.offsets.windows(2) {
let start = window[0];
let end = window[1];
let len = (end - start).to_usize().unwrap();
let b = values.drain(..len).collect::<Vec<u8>>();
if b.is_empty() {
builder.append_null();

if len != 0 {
builder
.try_append_view(block, start.as_usize() as u32, len as u32)
.unwrap();
} else {
builder.append_value(b);
builder.append_null();
}
}
builder
Expand Down

0 comments on commit c6359bf

Please sign in to comment.