Fixed UUID serde to match Presto Java

Added DecimalUtil::big for reversing int128_t byte order

velox/functions/prestosql/tests/UuidFunctionsTest.cpp

@@ -63,7 +63,7 @@ TEST_F(UuidFunctionsTest, castAsVarchar) {
   // Verify that CAST results as the same as boost::lexical_cast. We do not use
   // boost::lexical_cast to implement CAST because it is too slow.
   auto expected = makeFlatVector<std::string>(size, [&](auto row) {
-    const auto uuid = uuids->valueAt(row);
+    const auto uuid = DecimalUtil::big(uuids->valueAt(row));
 
     boost::uuids::uuid u;
     memcpy(&u, &uuid, 16);

velox/functions/prestosql/types/UuidType.cpp

@@ -75,7 +75,8 @@ class UuidCastOperator : public exec::CastOperator {
     const auto* uuids = input.as<SimpleVector<int128_t>>();
 
     context.applyToSelectedNoThrow(rows, [&](auto row) {
-      const auto uuid = uuids->valueAt(row);
+      // Make sure UUID bytes are big endian when building string
+      const auto uuid = DecimalUtil::big(uuids->valueAt(row));
 
       const uint8_t* uuidBytes = reinterpret_cast<const uint8_t*>(&uuid);
 
@@ -127,6 +128,9 @@ class UuidCastOperator : public exec::CastOperator {
       int128_t u;
       memcpy(&u, &uuid, 16);
 
+      // Value is big endian from boost, store as native byte-order
+      u = DecimalUtil::big(u);
+
       flatResult->set(row, u);
     });
   }

velox/serializers/PrestoSerializer.cpp

@@ -15,13 +15,15 @@
  */
 #include "velox/serializers/PrestoSerializer.h"
 
+#include <iostream>
 #include <optional>
 
 #include <folly/lang/Bits.h>
 
 #include "velox/common/base/Crc.h"
 #include "velox/common/base/RawVector.h"
 #include "velox/common/memory/ByteStream.h"
+#include "velox/functions/prestosql/types/UuidType.h"
 #include "velox/vector/BiasVector.h"
 #include "velox/vector/ComplexVector.h"
 #include "velox/vector/DictionaryVector.h"
@@ -442,6 +444,42 @@ void readDecimalValues(
   }
 }
 
+int128_t readUuidValue(ByteInputStream* source) {
+  // ByteInputStream does not support reading int128_t values.
+  // UUIDs are serialized as 2 int64 values with msb int64 value first.
+  auto high = source->read<uint64_t>();
+  auto low = source->read<uint64_t>();
+  return HugeInt::build(high, low);
+}
+
+void readUuidValues(
+    ByteInputStream* source,
+    vector_size_t size,
+    vector_size_t offset,
+    const BufferPtr& nulls,
+    vector_size_t nullCount,
+    const BufferPtr& values) {
+  auto rawValues = values->asMutable<int128_t>();
+  if (nullCount) {
+    checkValuesSize<int128_t>(values, nulls, size, offset);
+
+    int32_t toClear = offset;
+    bits::forEachSetBit(
+        nulls->as<uint64_t>(), offset, offset + size, [&](int32_t row) {
+          // Set the values between the last non-null and this to type default.
+          for (; toClear < row; ++toClear) {
+            rawValues[toClear] = 0;
+          }
+          rawValues[row] = readUuidValue(source);
+          toClear = row + 1;
+        });
+  } else {
+    for (int32_t row = 0; row < size; ++row) {
+      rawValues[offset + row] = readUuidValue(source);
+    }
+  }
+}
+
 /// When deserializing vectors under row vectors that introduce
 /// nulls, the child vector must have a gap at the place where a
 /// parent RowVector has a null. So, if there is a parent RowVector
@@ -565,6 +603,16 @@ void read(
         values);
     return;
   }
+  if (isUuidType(type)) {
+    readUuidValues(
+      source,
+      numNewValues,
+      resultOffset,
+      flatResult->nulls(),
+      nullCount,
+      values);
+    return;
+  }
   readValues<T>(
       source,
       numNewValues,
@@ -1364,6 +1412,7 @@ class VectorStream {
         useLosslessTimestamp_(opts.useLosslessTimestamp),
         nullsFirst_(opts.nullsFirst),
         isLongDecimal_(type_->isLongDecimal()),
+        isUuid_(isUuidType(type_)),
         opts_(opts),
         encoding_(getEncoding(encoding, vector)),
         nulls_(streamArena, true, true),
@@ -1709,6 +1758,10 @@ class VectorStream {
     return isLongDecimal_;
   }
 
+  bool isUuid() const {
+    return isUuid_;
+  }
+
   void clear() {
     encoding_ = std::nullopt;
     initializeHeader(typeToEncodingName(type_), *streamArena_);
@@ -1784,6 +1837,7 @@ class VectorStream {
   const bool useLosslessTimestamp_;
   const bool nullsFirst_;
   const bool isLongDecimal_;
+  const bool isUuid_;
   const SerdeOpts opts_;
   std::optional<VectorEncoding::Simple> encoding_;
   int32_t nonNullCount_{0};
@@ -1841,13 +1895,24 @@ FOLLY_ALWAYS_INLINE int128_t toJavaDecimalValue(int128_t value) {
   return value;
 }
 
+FOLLY_ALWAYS_INLINE int128_t toJavaUuidValue(int128_t value) {
+  // Presto Java UuidType uses java.util.UUID that expects 2 long values
+  // with most significant bits first, swap upper and lower to adjust.
+  auto low = HugeInt::upper(value);
+  auto high = HugeInt::lower(value);
+  return HugeInt::build(high, low);
+}
+
 template <>
 void VectorStream::append(folly::Range<const int128_t*> values) {
   for (auto& value : values) {
     int128_t val = value;
     if (isLongDecimal_) {
       val = toJavaDecimalValue(value);
     }
+    else if (isUuid_) {
+      val = toJavaUuidValue(value);
+    }
     values_.append<int128_t>(folly::Range(&val, 1));
   }
 }
@@ -2392,14 +2457,22 @@ void copyWords(
     const int32_t* indices,
     int32_t numIndices,
     const T* values,
-    bool isLongDecimal = false) {
+    bool isLongDecimal = false,
+    bool isUuid = false) {
   if (std::is_same_v<T, int128_t> && isLongDecimal) {
     for (auto i = 0; i < numIndices; ++i) {
       reinterpret_cast<int128_t*>(destination)[i] = toJavaDecimalValue(
           reinterpret_cast<const int128_t*>(values)[indices[i]]);
     }
     return;
   }
+  if (std::is_same_v<T, int128_t> && isUuid) {
+    for (auto i = 0; i < numIndices; ++i) {
+      reinterpret_cast<int128_t*>(destination)[i] = toJavaUuidValue(
+          reinterpret_cast<const int128_t*>(values)[indices[i]]);
+    }
+    return;
+  }
   for (auto i = 0; i < numIndices; ++i) {
     destination[i] = values[indices[i]];
   }
@@ -2412,9 +2485,10 @@ void copyWordsWithRows(
     const int32_t* indices,
     int32_t numIndices,
     const T* values,
-    bool isLongDecimal = false) {
+    bool isLongDecimal = false,
+    bool isUuid = false) {
   if (!indices) {
-    copyWords(destination, rows, numIndices, values, isLongDecimal);
+    copyWords(destination, rows, numIndices, values, isLongDecimal, isUuid);
     return;
   }
   if (std::is_same_v<T, int128_t> && isLongDecimal) {
@@ -2424,6 +2498,13 @@ void copyWordsWithRows(
     }
     return;
   }
+  else if (std::is_same_v<T, int128_t> && isUuid) {
+    for (auto i = 0; i < numIndices; ++i) {
+      reinterpret_cast<int128_t*>(destination)[i] = toJavaUuidValue(
+          reinterpret_cast<const int128_t*>(values)[rows[indices[i]]]);
+    }
+    return;
+  }
   for (auto i = 0; i < numIndices; ++i) {
     destination[i] = values[rows[indices[i]]];
   }
@@ -2484,7 +2565,8 @@ void appendNonNull(
         nonNullIndices,
         numNonNull,
         values,
-        stream->isLongDecimal());
+        stream->isLongDecimal(),
+        stream->isUuid());
   }
 }
 
@@ -2577,7 +2659,7 @@ void serializeFlatVector(
     AppendWindow<T> window(stream->values(), scratch);
     T* output = window.get(rows.size());
     copyWords(
-        output, rows.data(), rows.size(), rawValues, stream->isLongDecimal());
+        output, rows.data(), rows.size(), rawValues, stream->isLongDecimal(), stream->isUuid());
     return;
   }
 

velox/serializers/tests/PrestoSerializerTest.cpp

@@ -15,6 +15,7 @@
  */
 #include "velox/serializers/PrestoSerializer.h"
 #include <folly/Random.h>
+#include <functions/prestosql/types/UuidType.h>
 #include <gtest/gtest.h>
 #include <vector>
 #include "velox/common/base/tests/GTestUtils.h"
@@ -1054,6 +1055,23 @@ TEST_P(PrestoSerializerTest, longDecimal) {
   testRoundTrip(vector);
 }
 
+TEST_P(PrestoSerializerTest, uuid) {
+  std::vector<int128_t> uuidValues(200);
+
+  for (int row = 0; row < uuidValues.size(); row++) {
+    uuidValues[row] = (int128_t) 0xD1 << row % 120;
+  }
+  auto vector = makeFlatVector<int128_t>(uuidValues, UUID());
+
+  testRoundTrip(vector);
+
+  // Add some nulls.
+  for (auto i = 0; i < uuidValues.size(); i += 7) {
+    vector->setNull(i, true);
+  }
+  testRoundTrip(vector);
+}
+
 // Test that hierarchically encoded columns (rows) have their encodings
 // preserved by the PrestoBatchVectorSerializer.
 TEST_P(PrestoSerializerTest, encodingsBatchVectorSerializer) {

velox/type/DecimalUtil.cpp

@@ -125,4 +125,14 @@ int32_t DecimalUtil::maxStringViewSize(int precision, int scale) {
   return rowSize;
 }
 
+int128_t DecimalUtil::big(int128_t value) {
+  if (folly::kIsLittleEndian) {
+    auto upper = folly::Endian::big(HugeInt::upper(value));
+    auto lower = folly::Endian::big(HugeInt::lower(value));
+    return HugeInt::build(lower, upper);
+  } else {
+    return value;
+  }
+}
+
 } // namespace facebook::velox

velox/type/DecimalUtil.h

@@ -479,6 +479,13 @@ class DecimalUtil {
   /// @return The length of out.
   static int32_t toByteArray(int128_t value, char* out);
 
+  /// Reverse byte order of an int128_t, if native byte-order is little endian.
+  /// If native byte-order is big endian, the value will be unchanged. This
+  /// is similar to folly::Endian::big(), which does not support int128_t.
+  ///
+  /// \return A value with reversed byte-order for little endian platforms.
+  static int128_t big(int128_t value);
+
   static constexpr __uint128_t kOverflowMultiplier = ((__uint128_t)1 << 127);
 }; // DecimalUtil
 } // namespace facebook::velox