Skip to content

Commit

Permalink
Fix Presto serde bug when deserializing large payload
Browse files Browse the repository at this point in the history
  • Loading branch information
tanjialiang committed Oct 7, 2024
1 parent 96944d5 commit 470a67e
Show file tree
Hide file tree
Showing 2 changed files with 19 additions and 3 deletions.
13 changes: 10 additions & 3 deletions velox/serializers/PrestoSerializer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4220,12 +4220,19 @@ void PrestoVectorSerde::deserialize(
auto compressBuf = folly::IOBuf::create(header.compressedSize);
source->readBytes(compressBuf->writableData(), header.compressedSize);
compressBuf->append(header.compressedSize);

// Process chained uncompressed results IOBufs.
auto uncompress =
codec->uncompress(compressBuf.get(), header.uncompressedSize);
ByteRange byteRange{
uncompress->writableData(), (int32_t)uncompress->length(), 0};
std::vector<ByteRange> byteRanges;
auto* current = uncompress.get();
do {
byteRanges.push_back(
{current->writableData(), (int32_t)current->length(), 0});
current = current->next();
} while (current != uncompress.get());
auto uncompressedSource =
std::make_unique<BufferInputStream>(std::vector<ByteRange>{byteRange});
std::make_unique<BufferInputStream>(std::move(byteRanges));
readTopColumns(
*uncompressedSource, type, pool, *result, resultOffset, prestoOptions);
}
Expand Down
9 changes: 9 additions & 0 deletions velox/serializers/tests/PrestoSerializerTest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -775,6 +775,15 @@ TEST_P(PrestoSerializerTest, basic) {
testRoundTrip(rowVector);
}

TEST_P(PrestoSerializerTest, basicLarge) {
const vector_size_t numRows = 80'000;
auto rowVector = makeRowVector(
{makeFlatVector<int64_t>(numRows, [](vector_size_t row) { return row; }),
makeFlatVector<std::string>(
numRows, [](vector_size_t row) { return std::string(1024, 'x'); })});
testRoundTrip(rowVector);
}

/// Test serialization of a dictionary vector that adds nulls to the base
/// vector.
TEST_P(PrestoSerializerTest, dictionaryWithExtraNulls) {
Expand Down

0 comments on commit 470a67e

Please sign in to comment.