First working ip prefix

velox/docs/develop/types.rst

@@ -137,6 +137,7 @@ JSON                      VARCHAR
 TIMESTAMP WITH TIME ZONE  BIGINT
 UUID                      HUGEINT
 IPADDRESS                 HUGEINT
+IPPREFIX                  (HUGEINT, TINYINT)
 ========================  =====================
 
 TIMESTAMP WITH TIME ZONE represents a time point in milliseconds precision

velox/expression/tests/CustomTypeTest.cpp

@@ -216,7 +216,8 @@ TEST_F(CustomTypeTest, getCustomTypeNames) {
           "HYPERLOGLOG",
           "TIMESTAMP WITH TIME ZONE",
           "UUID",
-          "IPADDRESS"
+          "IPADDRESS",
+          "IPPREFIX"
       }),
       names);
 
@@ -231,6 +232,7 @@ TEST_F(CustomTypeTest, getCustomTypeNames) {
           "TIMESTAMP WITH TIME ZONE",
           "UUID",
           "IPADDRESS",
+          "IPPREFIX",
           "FANCY_INT",
       }),
       names);

velox/functions/prestosql/IPAddressFunctions.h

@@ -22,6 +22,7 @@
 #include "velox/functions/Registerer.h"
 #include "velox/functions/prestosql/types/UuidType.h"
 #include "velox/functions/prestosql/types/IPAddressType.h"
+#include "velox/functions/prestosql/types/IPPrefixType.h"
 #include "velox/functions/lib/string/StringImpl.h"
 
 #include <iostream>
@@ -47,49 +48,45 @@ template <typename T>
 struct IPPrefixFunction {
   VELOX_DEFINE_FUNCTION_TYPES(T);
 
-  FOLLY_ALWAYS_INLINE void call(out_type<Varchar>& result,
+  FOLLY_ALWAYS_INLINE void call(out_type<TheIPPrefix>& result,
                                 const arg_type<IPAddress>& ip,
                                 const arg_type<int8_t> prefixBits) {
-
-    std::string res;
+
     boost::asio::ip::address_v6::bytes_type addrBytes;
     memcpy(&addrBytes, &ip, 16);
+
+    // All IPs are stored as V6
     auto v6Addr = boost::asio::ip::make_address_v6(addrBytes);
-
+    boost::asio::ip::address_v6 v6CanonicalAddr;
+
+    //For return
+    int128_t canonicalAddrInt;
+
     // Presto stores prefixBits in one byte. Cast to unsigned
+    // Convert to V4/V6 respectively and create network to get canonical
+    // address as well as check validity of the prefix.
     if(v6Addr.is_v4_mapped()){
         auto v4Addr = boost::asio::ip::make_address_v4(boost::asio::ip::v4_mapped, v6Addr);
         auto v4Network = boost::asio::ip::make_network_v4(v4Addr, (uint8_t)prefixBits);
-        res = boost::lexical_cast<std::string>(v4Network.canonical());
+        v6CanonicalAddr = boost::asio::ip::make_address_v6(boost::asio::ip::v4_mapped, v4Network.canonical().address());
       }else{
         auto v6Network = boost::asio::ip::make_network_v6(v6Addr, (uint8_t)prefixBits);
-        res = boost::lexical_cast<std::string>(v6Network.canonical());
+        v6CanonicalAddr = v6Network.canonical().address();
     }
 
-
-    /*
-    folly::ByteArray16 addrBytes;
-    folly::IPAddressV6 addr(addrBytes);
-
-    // Presto stores prefixBits in one byte. Cast to unsigned
-    if(addr.isIPv4Mapped()){
-      res = addr.createIPv4().mask((uint8_t)prefixBits).str();
-    }else{
-      res = addr.mask((uint8_t)prefixBits).str();
-    }*/
+    auto canonicalBytes = v6CanonicalAddr.to_bytes();
+    memcpy(&canonicalAddrInt, &canonicalBytes, 16);
 
-
-
-    result.reserve(res.length());
-    result.append(res);
+    result =  std::make_shared<IPPrefix>(canonicalAddrInt, (uint8_t)prefixBits);
     }
 };
 
 
 inline void registerIPAddressFunctions(const std::string& prefix) {
   registerIPAddressType();
+  registerIPPrefixType();
   registerFunction<IPAddressFunction, IPAddress>({prefix + "ipaddress"});
-  registerFunction<IPPrefixFunction, Varchar, IPAddress, int8_t>({prefix + "ip_prefix"});
+  registerFunction<IPPrefixFunction, TheIPPrefix, IPAddress, int8_t>({prefix + "ip_prefix"});
 
 }
 

velox/functions/prestosql/TypeOf.cpp

@@ -19,6 +19,7 @@
 #include "velox/functions/prestosql/types/TimestampWithTimeZoneType.h"
 #include "velox/functions/prestosql/types/UuidType.h"
 #include "velox/functions/prestosql/types/IPAddressType.h"
+#include "velox/functions/prestosql/types/IPPrefixType.h"
 
 namespace facebook::velox::functions {
 namespace {
@@ -108,6 +109,11 @@ std::string typeName(const TypePtr& type) {
     }
     case TypeKind::UNKNOWN:
       return "unknown";
+    case TypeKind::OPAQUE:
+      if (isIPPrefixType(type)) {
+        return "ipprefix";
+      }
+      return "opaque";
     default:
       VELOX_UNSUPPORTED("Unsupported type: {}", type->toString())
   }

velox/functions/prestosql/tests/CMakeLists.txt

@@ -16,93 +16,93 @@ add_subdirectory(utils)
 
 add_executable(
   velox_functions_test
-  ArithmeticTest.cpp
-  ArrayAllMatchTest.cpp
-  ArrayAnyMatchTest.cpp
-  ArrayAverageTest.cpp
-  ArrayCombinationsTest.cpp
-  ArrayConcatTest.cpp
-  ArrayConstructorTest.cpp
-  ArrayContainsTest.cpp
-  ArrayDistinctTest.cpp
-  ArrayDuplicatesTest.cpp
-  ArrayExceptTest.cpp
-  ArrayFilterTest.cpp
-  ArrayFrequencyTest.cpp
-  ArrayFlattenTest.cpp
-  ArrayHasDuplicatesTest.cpp
-  ArrayIntersectTest.cpp
-  ArrayJoinTest.cpp
-  ArrayMaxTest.cpp
-  ArrayMinTest.cpp
-  ArrayNGramsTest.cpp
-  ArrayNoneMatchTest.cpp
-  ArrayNormalizeTest.cpp
-  ArrayPositionTest.cpp
-  ArrayRemoveNullsTest.cpp
-  ArrayRemoveTest.cpp
-  ArrayShuffleTest.cpp
-  ArraySortTest.cpp
-  ArraysOverlapTest.cpp
-  ArraySumTest.cpp
-  ArrayTrimTest.cpp
-  ArrayUnionTest.cpp
-  ArgGeneratorTest.cpp
-  BinaryFunctionsTest.cpp
-  BitwiseTest.cpp
-  CardinalityTest.cpp
-  CeilFloorTest.cpp
-  ComparisonsTest.cpp
-  DateTimeFunctionsTest.cpp
-  DecimalArithmeticTest.cpp
-  ElementAtTest.cpp
-  FindFirstTest.cpp
-  FromUtf8Test.cpp
-  GreatestLeastTest.cpp
-  HyperLogLogCastTest.cpp
-  HyperLogLogFunctionsTest.cpp
+#  ArithmeticTest.cpp
+#  ArrayAllMatchTest.cpp
+  # ArrayAnyMatchTest.cpp
+  # ArrayAverageTest.cpp
+  # ArrayCombinationsTest.cpp
+  # ArrayConcatTest.cpp
+  # ArrayConstructorTest.cpp
+  # ArrayContainsTest.cpp
+  # ArrayDistinctTest.cpp
+  # ArrayDuplicatesTest.cpp
+  # ArrayExceptTest.cpp
+  # ArrayFilterTest.cpp
+  # ArrayFrequencyTest.cpp
+  # ArrayFlattenTest.cpp
+  # ArrayHasDuplicatesTest.cpp
+  # ArrayIntersectTest.cpp
+  # ArrayJoinTest.cpp
+  # ArrayMaxTest.cpp
+  # ArrayMinTest.cpp
+  # ArrayNGramsTest.cpp
+  # ArrayNoneMatchTest.cpp
+  # ArrayNormalizeTest.cpp
+  # ArrayPositionTest.cpp
+  # ArrayRemoveNullsTest.cpp
+  # ArrayRemoveTest.cpp
+  # ArrayShuffleTest.cpp
+  # ArraySortTest.cpp
+  # ArraysOverlapTest.cpp
+  # ArraySumTest.cpp
+  # ArrayTrimTest.cpp
+  # ArrayUnionTest.cpp
+  # ArgGeneratorTest.cpp
+  # BinaryFunctionsTest.cpp
+  # BitwiseTest.cpp
+  # CardinalityTest.cpp
+  # CeilFloorTest.cpp
+  # ComparisonsTest.cpp
+  # DateTimeFunctionsTest.cpp
+  # DecimalArithmeticTest.cpp
+  # ElementAtTest.cpp
+  # FindFirstTest.cpp
+  # FromUtf8Test.cpp
+  # GreatestLeastTest.cpp
+  # HyperLogLogCastTest.cpp
+  # HyperLogLogFunctionsTest.cpp
   IPAddressFunctionsTest.cpp
-  InPredicateTest.cpp
-  JsonCastTest.cpp
-  JsonExtractScalarTest.cpp
-  JsonFunctionsTest.cpp
-  MapEntriesTest.cpp
-  MapFilterTest.cpp
-  MapFromEntriesTest.cpp
-  MapNormalizeTest.cpp
-  MapTopNTest.cpp
-  MapKeysAndValuesTest.cpp
-  MapMatchTest.cpp
-  MapTest.cpp
-  MapZipWithTest.cpp
-  MultimapFromEntriesTest.cpp
-  NotTest.cpp
-  ProbabilityTest.cpp
-  RandTest.cpp
-  ReduceTest.cpp
-  RegexpReplaceTest.cpp
-  ReverseTest.cpp
-  RoundTest.cpp
-  ScalarFunctionRegTest.cpp
-  SequenceTest.cpp
-  SimpleComparisonMatcherTest.cpp
-  SliceTest.cpp
-  SplitTest.cpp
-  SplitToMapTest.cpp
-  StringFunctionsTest.cpp
-  TimestampWithTimeZoneCastTest.cpp
-  TransformKeysTest.cpp
-  TransformTest.cpp
-  TransformValuesTest.cpp
-  TrimFunctionsTest.cpp
-  TypeOfTest.cpp
-  URLFunctionsTest.cpp
-  Utf8Test.cpp
-  UuidFunctionsTest.cpp
-  WidthBucketArrayTest.cpp
-  WordStemTest.cpp
-  ZipTest.cpp
-  ZipWithTest.cpp)
+  # InPredicateTest.cpp
+  # JsonCastTest.cpp
+  # JsonExtractScalarTest.cpp
+  # JsonFunctionsTest.cpp
+  # MapEntriesTest.cpp
+  # MapFilterTest.cpp
+  # MapFromEntriesTest.cpp
+  # MapNormalizeTest.cpp
+  # MapTopNTest.cpp
+  # MapKeysAndValuesTest.cpp
+  # MapMatchTest.cpp
+  # MapTest.cpp
+  # MapZipWithTest.cpp
+  # MultimapFromEntriesTest.cpp
+  # NotTest.cpp
+  # ProbabilityTest.cpp
+  # RandTest.cpp
+  # ReduceTest.cpp
+  # RegexpReplaceTest.cpp
+  # ReverseTest.cpp
+  # RoundTest.cpp
+  # ScalarFunctionRegTest.cpp
+  # SequenceTest.cpp
+  # SimpleComparisonMatcherTest.cpp
+  # SliceTest.cpp
+  # SplitTest.cpp
+  # SplitToMapTest.cpp
+  # StringFunctionsTest.cpp
+  # TimestampWithTimeZoneCastTest.cpp
+  # TransformKeysTest.cpp
+  # TransformTest.cpp
+  # TransformValuesTest.cpp
+  # TrimFunctionsTest.cpp
+  # TypeOfTest.cpp
+  # URLFunctionsTest.cpp
+  # Utf8Test.cpp
+  # UuidFunctionsTest.cpp
+  # WidthBucketArrayTest.cpp
+  # WordStemTest.cpp
+  # ZipTest.cpp
+   ZipWithTest.cpp)
 
 add_test(velox_functions_test velox_functions_test)
 

velox/functions/prestosql/tests/IPAddressFunctionsTest.cpp

@@ -24,7 +24,7 @@ namespace {
 class IPAddressTest : public functions::test::FunctionBaseTest {
 protected:
   std::optional<std::string> getIPPrefix(const std::optional<std::string> input, std::optional<int8_t> mask) {
-    auto result = evaluateOnce<std::string>("ip_prefix(cast(c0 as ipaddress), c1)", input, mask);
+    auto result = evaluateOnce<std::string>("cast(ip_prefix(cast(c0 as ipaddress), c1) as varchar)", input, mask);
     return result;
   }
 };
@@ -85,6 +85,22 @@ TEST_F(IPAddressTest, IPPrefixv6) {
 
 }
 
+TEST_F(IPAddressTest, castRoundTripPrefix) {
+  auto strings = makeFlatVector<std::string>({
+      "87a0:ce14:8989:44c9:826e:b4d8:73f9:1542/48",
+      "7cd6:bcec:1216:5c20:4b67:b1bd:173:ced/5",
+      "192.128.0.0/5"
+  });
+
+  auto ipprefixes = evaluate("cast(c0 as ipprefix)", makeRowVector({strings}));
+  auto stringsCopy = evaluate("cast(c0 as varchar)", makeRowVector({ipprefixes}));
+  auto ipprefixesCopy = evaluate("cast(c0 as ipprefix)", makeRowVector({stringsCopy}));
+
+  // assertEqualVectors are comparing the shared pointers so cannot compare 
+  // ipprefixes and ipprefixesCopy
+  velox::test::assertEqualVectors(strings, stringsCopy);
+
+}
 
 }
 

velox/functions/prestosql/types/CMakeLists.txt

@@ -13,7 +13,7 @@
 # limitations under the License.
 add_library(velox_presto_types HyperLogLogType.cpp JsonType.cpp
                                TimestampWithTimeZoneType.cpp UuidType.cpp
-                               IPAddressType.cpp) #IPPrefixType.cpp)
+                               IPAddressType.cpp IPPrefixType.cpp)
 
 target_link_libraries(
   velox_presto_types velox_memory velox_expression velox_functions_util

velox/functions/prestosql/types/IPAddressType.cpp

@@ -79,7 +79,6 @@ class IPAddressCastOperator : public exec::CastOperator {
       memcpy(&addrBytes, &intAddr, 16);
 
       auto v6Addr = boost::asio::ip::make_address_v6(addrBytes);
-      std::cerr << "Before conversion: " << boost::lexical_cast<std::string>(v6Addr) << std::endl;
 
       if(v6Addr.is_v4_mapped()){
         auto v4Addr = boost::asio::ip::make_address_v4(boost::asio::ip::v4_mapped, v6Addr);
@@ -88,19 +87,6 @@ class IPAddressCastOperator : public exec::CastOperator {
         s = boost::lexical_cast<std::string>(v6Addr);
       }
 
-      std::cerr << "Casted to string: " << s << std::endl;
-
-     /*
-      folly::ByteArray16 addrBytes;
-      std::string s;
-      memcpy(&addrBytes, &intAddr, 16);
-      IPAddressV6 addr(addrBytes);
-      if(addr.isIPv4Mapped()){
-        s = addr.createIPv4().str();
-      }else{
-        s = addr.str();
-      }*/
-
       exec::StringWriter<false> result(flatResult, row);
       result.append(s);
       result.finalize();
@@ -118,13 +104,6 @@ class IPAddressCastOperator : public exec::CastOperator {
     context.applyToSelectedNoThrow(rows, [&](auto row) {
       const auto ipAddressString = ipAddressStrings->valueAt(row);
       boost::asio::ip::address_v6::bytes_type addrBytes;
-      /*
-      // Create address and format as V6
-      folly::IPAddress addr(ipAddressString);
-      folly::IPAddressV6 fmtAddr = folly::IPAddress::createIPv6(addr);
-      int128_t intAddr;
-      memcpy(&intAddr, fmtAddr.bytes(), 16);
-      */
       auto addr =  boost::asio::ip::make_address(ipAddressString);
       int128_t intAddr;
       if(addr.is_v4()){