From d23db6f5cf9c5966ca3558a24a0646d58594b5ed Mon Sep 17 00:00:00 2001 From: Bryan Cutler Date: Fri, 1 Nov 2024 11:53:36 -0700 Subject: [PATCH] Fixed UUID serde to match Presto Java Summary: This fixes the PrestoSerializer to put UUID values in the correct format that is expected by Presto Java so that the values will match those from a Java worker. First, when converting UUID to/from string, the values are no longer in big endian format (as taken from boost::uuid) and are instead stored as a little endian in an int128_t. Secondly, Presto Java will read UUID values from an Int128ArrayBlock with the first value as the most significant bits. To correct this, the upper/lower parts of the int128_t are swapped during serialization/deserialization. A unit test for checking roundtrip UUID serializaiton was added and manual testing of Presto with a native worker to verify the problem from the issue description is fixed. From prestodb/presto#23311 --- .../prestosql/tests/UuidFunctionsTest.cpp | 2 +- velox/functions/prestosql/types/UuidType.cpp | 6 +- velox/serializers/PrestoSerializer.cpp | 77 ++++++++++++++++++- .../tests/PrestoSerializerTest.cpp | 14 ++++ velox/type/DecimalUtil.h | 15 ++++ 5 files changed, 110 insertions(+), 4 deletions(-) diff --git a/velox/functions/prestosql/tests/UuidFunctionsTest.cpp b/velox/functions/prestosql/tests/UuidFunctionsTest.cpp index a66a9ff5adbb2..4fc636d966f35 100644 --- a/velox/functions/prestosql/tests/UuidFunctionsTest.cpp +++ b/velox/functions/prestosql/tests/UuidFunctionsTest.cpp @@ -63,7 +63,7 @@ TEST_F(UuidFunctionsTest, castAsVarchar) { // Verify that CAST results as the same as boost::lexical_cast. We do not use // boost::lexical_cast to implement CAST because it is too slow. auto expected = makeFlatVector(size, [&](auto row) { - const auto uuid = uuids->valueAt(row); + const auto uuid = DecimalUtil::bigEndian(uuids->valueAt(row)); boost::uuids::uuid u; memcpy(&u, &uuid, 16); diff --git a/velox/functions/prestosql/types/UuidType.cpp b/velox/functions/prestosql/types/UuidType.cpp index 5aa420113a69d..b97d5f3d57353 100644 --- a/velox/functions/prestosql/types/UuidType.cpp +++ b/velox/functions/prestosql/types/UuidType.cpp @@ -75,7 +75,8 @@ class UuidCastOperator : public exec::CastOperator { const auto* uuids = input.as>(); context.applyToSelectedNoThrow(rows, [&](auto row) { - const auto uuid = uuids->valueAt(row); + // Ensure UUID bytes are big endian when building the string. + const auto uuid = DecimalUtil::bigEndian(uuids->valueAt(row)); const uint8_t* uuidBytes = reinterpret_cast(&uuid); @@ -127,6 +128,9 @@ class UuidCastOperator : public exec::CastOperator { int128_t u; memcpy(&u, &uuid, 16); + // Convert a big endian value from Boost to native byte-order. + u = DecimalUtil::bigEndian(u); + flatResult->set(row, u); }); } diff --git a/velox/serializers/PrestoSerializer.cpp b/velox/serializers/PrestoSerializer.cpp index 2ae3207457aa6..403d3d3516346 100644 --- a/velox/serializers/PrestoSerializer.cpp +++ b/velox/serializers/PrestoSerializer.cpp @@ -23,6 +23,7 @@ #include "velox/common/base/IOUtils.h" #include "velox/common/base/RawVector.h" #include "velox/common/memory/ByteStream.h" +#include "velox/functions/prestosql/types/UuidType.h" #include "velox/vector/BiasVector.h" #include "velox/vector/ComplexVector.h" #include "velox/vector/DictionaryVector.h" @@ -429,6 +430,42 @@ void readDecimalValues( if (nullCount) { checkValuesSize(values, nulls, size, offset); + vector_size_t toClear = offset; + bits::forEachSetBit( + nulls->as(), offset, offset + size, [&](vector_size_t row) { + // Set the values between the last non-null and this to type default. + for (; toClear < row; ++toClear) { + rawValues[toClear] = 0; + } + rawValues[row] = readJavaDecimal(source); + toClear = row + 1; + }); + } else { + for (vector_size_t row = 0; row < size; ++row) { + rawValues[offset + row] = readJavaDecimal(source); + } + } +} + +int128_t readUuidValue(ByteInputStream* source) { + // ByteInputStream does not support reading int128_t values. + // UUIDs are serialized as 2 uint64 values with msb value first. + auto high = folly::Endian::big(source->read()); + auto low = folly::Endian::big(source->read()); + return HugeInt::build(high, low); +} + +void readUuidValues( + ByteInputStream* source, + vector_size_t size, + vector_size_t offset, + const BufferPtr& nulls, + vector_size_t nullCount, + const BufferPtr& values) { + auto rawValues = values->asMutable(); + if (nullCount) { + checkValuesSize(values, nulls, size, offset); + int32_t toClear = offset; bits::forEachSetBit( nulls->as(), offset, offset + size, [&](int32_t row) { @@ -436,12 +473,12 @@ void readDecimalValues( for (; toClear < row; ++toClear) { rawValues[toClear] = 0; } - rawValues[row] = readJavaDecimal(source); + rawValues[row] = readUuidValue(source); toClear = row + 1; }); } else { for (int32_t row = 0; row < size; ++row) { - rawValues[offset + row] = readJavaDecimal(source); + rawValues[offset + row] = readUuidValue(source); } } } @@ -569,6 +606,16 @@ void read( values); return; } + if (isUuidType(type)) { + readUuidValues( + source, + numNewValues, + resultOffset, + flatResult->nulls(), + nullCount, + values); + return; + } readValues( source, numNewValues, @@ -1427,6 +1474,7 @@ class VectorStream { useLosslessTimestamp_(opts.useLosslessTimestamp), nullsFirst_(opts.nullsFirst), isLongDecimal_(type_->isLongDecimal()), + isUuid_(isUuidType(type_)), opts_(opts), encoding_(getEncoding(encoding, vector)), nulls_(streamArena, true, true), @@ -1778,6 +1826,10 @@ class VectorStream { return isLongDecimal_; } + bool isUuid() const { + return isUuid_; + } + void clear() { encoding_ = std::nullopt; initializeHeader(typeToEncodingName(type_), *streamArena_); @@ -1858,6 +1910,7 @@ class VectorStream { const bool useLosslessTimestamp_; const bool nullsFirst_; const bool isLongDecimal_; + const bool isUuid_; const SerdeOpts opts_; std::optional encoding_; int32_t nonNullCount_{0}; @@ -1915,12 +1968,21 @@ FOLLY_ALWAYS_INLINE int128_t toJavaDecimalValue(int128_t value) { return value; } +FOLLY_ALWAYS_INLINE int128_t toJavaUuidValue(int128_t value) { + // Presto Java UuidType expects 2 long values with MSB first. + // int128 will be serialized with [lower, upper], so swap here + // to adjust the order. + return DecimalUtil::bigEndian(value); +} + template <> void VectorStream::append(folly::Range values) { for (auto& value : values) { int128_t val = value; if (isLongDecimal_) { val = toJavaDecimalValue(value); + } else if (isUuid_) { + val = toJavaUuidValue(value); } values_.append(folly::Range(&val, 1)); } @@ -2595,6 +2657,14 @@ void appendNonNull( numNonNull, values, toJavaDecimalValue); + } else if (stream->isUuid()) { + copyWordsWithRows( + output, + rows.data(), + nonNullIndices, + numNonNull, + values, + toJavaUuidValue); } else { copyWordsWithRows( output, rows.data(), nonNullIndices, numNonNull, values); @@ -2678,6 +2748,9 @@ void serializeFlatVector( if (stream->isLongDecimal()) { copyWords( output, rows.data(), rows.size(), rawValues, toJavaDecimalValue); + } else if (stream->isUuid()) { + copyWords( + output, rows.data(), rows.size(), rawValues, toJavaUuidValue); } else { copyWords(output, rows.data(), rows.size(), rawValues); } diff --git a/velox/serializers/tests/PrestoSerializerTest.cpp b/velox/serializers/tests/PrestoSerializerTest.cpp index ba1d3362720e9..49cdbdbef0f8f 100644 --- a/velox/serializers/tests/PrestoSerializerTest.cpp +++ b/velox/serializers/tests/PrestoSerializerTest.cpp @@ -16,6 +16,7 @@ #include "velox/serializers/PrestoSerializer.h" #include #include +#include #include #include #include "velox/common/base/tests/GTestUtils.h" @@ -1149,6 +1150,19 @@ TEST_P(PrestoSerializerTest, longDecimal) { testRoundTrip(vector); } +TEST_P(PrestoSerializerTest, uuid) { + auto vector = makeFlatVector( + 200, [](vector_size_t row) { return (int128_t)0xD1 << row % 120; }); + + testRoundTrip(vector); + + // Add some nulls. + for (auto i = 0; i < vector->size(); i += 7) { + vector->setNull(i, true); + } + testRoundTrip(vector); +} + // Test that hierarchically encoded columns (rows) have their encodings // preserved by the PrestoBatchVectorSerializer. TEST_P(PrestoSerializerTest, encodingsBatchVectorSerializer) { diff --git a/velox/type/DecimalUtil.h b/velox/type/DecimalUtil.h index dc935b94cdc95..0b91d21b02dae 100644 --- a/velox/type/DecimalUtil.h +++ b/velox/type/DecimalUtil.h @@ -479,6 +479,21 @@ class DecimalUtil { /// @return The length of out. static int32_t toByteArray(int128_t value, char* out); + /// Reverse byte order of an int128_t if native byte-order is little endian. + /// If native byte-order is big endian, the value will be unchanged. This + /// is similar to folly::Endian::big(), which does not support int128_t. + /// + /// \return A value with reversed byte-order for little endian platforms. + inline static int128_t bigEndian(int128_t value) { + if (folly::kIsLittleEndian) { + auto upper = folly::Endian::big(HugeInt::upper(value)); + auto lower = folly::Endian::big(HugeInt::lower(value)); + return HugeInt::build(lower, upper); + } else { + return value; + } + } + static constexpr __uint128_t kOverflowMultiplier = ((__uint128_t)1 << 127); }; // DecimalUtil } // namespace facebook::velox