diff --git a/velox/expression/tests/CustomTypeTest.cpp b/velox/expression/tests/CustomTypeTest.cpp index 7d48be178a8d4..26540cd2fdd6a 100644 --- a/velox/expression/tests/CustomTypeTest.cpp +++ b/velox/expression/tests/CustomTypeTest.cpp @@ -217,8 +217,7 @@ TEST_F(CustomTypeTest, getCustomTypeNames) { "TIMESTAMP WITH TIME ZONE", "UUID", "IPADDRESS", - "IPPREFIX" - }), + "IPPREFIX"}), names); ASSERT_TRUE(registerCustomType( diff --git a/velox/functions/prestosql/IPAddressFunctions.h b/velox/functions/prestosql/IPAddressFunctions.h index c631e6644a7cb..3db3b8f96afb3 100644 --- a/velox/functions/prestosql/IPAddressFunctions.h +++ b/velox/functions/prestosql/IPAddressFunctions.h @@ -20,10 +20,10 @@ #include "velox/functions/Macros.h" #include "velox/functions/Registerer.h" -#include "velox/functions/prestosql/types/UuidType.h" +#include "velox/functions/lib/string/StringImpl.h" #include "velox/functions/prestosql/types/IPAddressType.h" #include "velox/functions/prestosql/types/IPPrefixType.h" -#include "velox/functions/lib/string/StringImpl.h" +#include "velox/functions/prestosql/types/UuidType.h" #include @@ -43,15 +43,14 @@ struct IPAddressFunction { boost::uuids::random_generator generator_; }; - template struct IPPrefixFunction { VELOX_DEFINE_FUNCTION_TYPES(T); - FOLLY_ALWAYS_INLINE void call(out_type& result, - const arg_type& ip, - const arg_type prefixBits) { - + FOLLY_ALWAYS_INLINE void call( + out_type& result, + const arg_type& ip, + const arg_type prefixBits) { boost::asio::ip::address_v6::bytes_type addrBytes; memcpy(&addrBytes, &ip, 16); @@ -59,35 +58,38 @@ struct IPPrefixFunction { auto v6Addr = boost::asio::ip::make_address_v6(addrBytes); boost::asio::ip::address_v6 v6CanonicalAddr; - //For return + // For return int128_t canonicalAddrInt; // Presto stores prefixBits in one byte. Cast to unsigned // Convert to V4/V6 respectively and create network to get canonical // address as well as check validity of the prefix. - if(v6Addr.is_v4_mapped()){ - auto v4Addr = boost::asio::ip::make_address_v4(boost::asio::ip::v4_mapped, v6Addr); - auto v4Network = boost::asio::ip::make_network_v4(v4Addr, (uint8_t)prefixBits); - v6CanonicalAddr = boost::asio::ip::make_address_v6(boost::asio::ip::v4_mapped, v4Network.canonical().address()); - }else{ - auto v6Network = boost::asio::ip::make_network_v6(v6Addr, (uint8_t)prefixBits); - v6CanonicalAddr = v6Network.canonical().address(); + if (v6Addr.is_v4_mapped()) { + auto v4Addr = + boost::asio::ip::make_address_v4(boost::asio::ip::v4_mapped, v6Addr); + auto v4Network = + boost::asio::ip::make_network_v4(v4Addr, (uint8_t)prefixBits); + v6CanonicalAddr = boost::asio::ip::make_address_v6( + boost::asio::ip::v4_mapped, v4Network.canonical().address()); + } else { + auto v6Network = + boost::asio::ip::make_network_v6(v6Addr, (uint8_t)prefixBits); + v6CanonicalAddr = v6Network.canonical().address(); } - + auto canonicalBytes = v6CanonicalAddr.to_bytes(); memcpy(&canonicalAddrInt, &canonicalBytes, 16); - - result = std::make_shared(canonicalAddrInt, (uint8_t)prefixBits); - } -}; + result = std::make_shared(canonicalAddrInt, (uint8_t)prefixBits); + } +}; inline void registerIPAddressFunctions(const std::string& prefix) { registerIPAddressType(); registerIPPrefixType(); registerFunction({prefix + "ipaddress"}); - registerFunction({prefix + "ip_prefix"}); - + registerFunction( + {prefix + "ip_prefix"}); } } // namespace facebook::velox::functions diff --git a/velox/functions/prestosql/TypeOf.cpp b/velox/functions/prestosql/TypeOf.cpp index e043db8842782..0927dd4821afb 100644 --- a/velox/functions/prestosql/TypeOf.cpp +++ b/velox/functions/prestosql/TypeOf.cpp @@ -15,11 +15,11 @@ */ #include "velox/expression/VectorFunction.h" #include "velox/functions/prestosql/types/HyperLogLogType.h" +#include "velox/functions/prestosql/types/IPAddressType.h" +#include "velox/functions/prestosql/types/IPPrefixType.h" #include "velox/functions/prestosql/types/JsonType.h" #include "velox/functions/prestosql/types/TimestampWithTimeZoneType.h" #include "velox/functions/prestosql/types/UuidType.h" -#include "velox/functions/prestosql/types/IPAddressType.h" -#include "velox/functions/prestosql/types/IPPrefixType.h" namespace facebook::velox::functions { namespace { @@ -56,8 +56,7 @@ std::string typeName(const TypePtr& type) { case TypeKind::HUGEINT: { if (isUuidType(type)) { return "uuid"; - } - else if (isIPAddressType(type)) { + } else if (isIPAddressType(type)) { return "ipaddress"; } VELOX_USER_CHECK( diff --git a/velox/functions/prestosql/registration/RegistrationFunctions.cpp b/velox/functions/prestosql/registration/RegistrationFunctions.cpp index 8ba86c72345ed..4f0e66875fac0 100644 --- a/velox/functions/prestosql/registration/RegistrationFunctions.cpp +++ b/velox/functions/prestosql/registration/RegistrationFunctions.cpp @@ -14,8 +14,8 @@ * limitations under the License. */ #include -#include "velox/functions/prestosql/UuidFunctions.h" #include "velox/functions/prestosql/IPAddressFunctions.h" +#include "velox/functions/prestosql/UuidFunctions.h" namespace facebook::velox::functions { diff --git a/velox/functions/prestosql/tests/IPAddressFunctionsTest.cpp b/velox/functions/prestosql/tests/IPAddressFunctionsTest.cpp index 3f9e97a1994b6..51a6c3649bb59 100644 --- a/velox/functions/prestosql/tests/IPAddressFunctionsTest.cpp +++ b/velox/functions/prestosql/tests/IPAddressFunctionsTest.cpp @@ -22,9 +22,12 @@ namespace facebook::velox::functions::prestosql { namespace { class IPAddressTest : public functions::test::FunctionBaseTest { -protected: - std::optional getIPPrefix(const std::optional input, std::optional mask) { - auto result = evaluateOnce("cast(ip_prefix(cast(c0 as ipaddress), c1) as varchar)", input, mask); + protected: + std::optional getIPPrefix( + const std::optional input, + std::optional mask) { + auto result = evaluateOnce( + "cast(ip_prefix(cast(c0 as ipaddress), c1) as varchar)", input, mask); return result; } }; @@ -43,65 +46,80 @@ TEST_F(IPAddressTest, castAsVarchar) { } TEST_F(IPAddressTest, castRoundTrip) { - auto strings = makeFlatVector({ - "87a0:ce14:8989:44c9:826e:b4d8:73f9:1542", - "7cd6:bcec:1216:5c20:4b67:b1bd:173:ced", - "192.128.0.0" - }); - - auto ipaddresses = evaluate("cast(c0 as ipaddress)", makeRowVector({strings})); - auto stringsCopy = evaluate("cast(c0 as varchar)", makeRowVector({ipaddresses})); - auto ipaddressesCopy = evaluate("cast(c0 as ipaddress)", makeRowVector({stringsCopy})); + auto strings = makeFlatVector( + {"87a0:ce14:8989:44c9:826e:b4d8:73f9:1542", + "7cd6:bcec:1216:5c20:4b67:b1bd:173:ced", + "192.128.0.0"}); + + auto ipaddresses = + evaluate("cast(c0 as ipaddress)", makeRowVector({strings})); + auto stringsCopy = + evaluate("cast(c0 as varchar)", makeRowVector({ipaddresses})); + auto ipaddressesCopy = + evaluate("cast(c0 as ipaddress)", makeRowVector({stringsCopy})); velox::test::assertEqualVectors(strings, stringsCopy); velox::test::assertEqualVectors(ipaddresses, ipaddressesCopy); } TEST_F(IPAddressTest, IPPrefixv4) { - - EXPECT_EQ("10.0.0.0/8", getIPPrefix("10.135.23.12",8)); + EXPECT_EQ("10.0.0.0/8", getIPPrefix("10.135.23.12", 8)); EXPECT_EQ("192.128.0.0/9", getIPPrefix("192.168.255.255", 9)); - EXPECT_EQ("192.168.255.255/32", getIPPrefix("192.168.255.255",32)); - EXPECT_EQ("0.0.0.0/0", getIPPrefix("192.168.255.255",0)); - - EXPECT_THROW(getIPPrefix("12.483.09.1",8), VeloxUserError); - EXPECT_THROW(getIPPrefix("10.135.23.12.12",8), VeloxUserError); - EXPECT_THROW(getIPPrefix("10.135.23",8), VeloxUserError); - EXPECT_THROW(getIPPrefix("12.135.23.12",-1), VeloxUserError); - EXPECT_THROW(getIPPrefix("10.135.23.12",33), VeloxUserError); + EXPECT_EQ("192.168.255.255/32", getIPPrefix("192.168.255.255", 32)); + EXPECT_EQ("0.0.0.0/0", getIPPrefix("192.168.255.255", 0)); + + EXPECT_THROW(getIPPrefix("12.483.09.1", 8), VeloxUserError); + EXPECT_THROW(getIPPrefix("10.135.23.12.12", 8), VeloxUserError); + EXPECT_THROW(getIPPrefix("10.135.23", 8), VeloxUserError); + EXPECT_THROW(getIPPrefix("12.135.23.12", -1), VeloxUserError); + EXPECT_THROW(getIPPrefix("10.135.23.12", 33), VeloxUserError); } TEST_F(IPAddressTest, IPPrefixv6) { - EXPECT_EQ("2001:db8:85a3::/48", getIPPrefix("2001:0db8:85a3:0001:0001:8a2e:0370:7334", 48)); - EXPECT_EQ("2001:db8:85a3::/52", getIPPrefix("2001:0db8:85a3:0001:0001:8a2e:0370:7334", 52)); - EXPECT_EQ("2001:db8:85a3:1:1:8a2e:370:7334/128", getIPPrefix("2001:0db8:85a3:0001:0001:8a2e:0370:7334", 128)); + EXPECT_EQ( + "2001:db8:85a3::/48", + getIPPrefix("2001:0db8:85a3:0001:0001:8a2e:0370:7334", 48)); + EXPECT_EQ( + "2001:db8:85a3::/52", + getIPPrefix("2001:0db8:85a3:0001:0001:8a2e:0370:7334", 52)); + EXPECT_EQ( + "2001:db8:85a3:1:1:8a2e:370:7334/128", + getIPPrefix("2001:0db8:85a3:0001:0001:8a2e:0370:7334", 128)); EXPECT_EQ("::/0", getIPPrefix("2001:0db8:85a3:0001:0001:8a2e:0370:7334", 0)); - EXPECT_THROW(getIPPrefix("q001:0db8:85a3:0001:0001:8a2e:0370:7334", 8), VeloxUserError); - EXPECT_THROW(getIPPrefix("2001:0db8:85a3:542e:0001:0001:8a2e:0370:7334", 8), VeloxUserError); - EXPECT_THROW(getIPPrefix("2001:0db8:85a3:0001:0001:8a2e:0370", 8), VeloxUserError); - EXPECT_THROW(getIPPrefix("2001:0db8:85a3:0001:0001:8a2e:0370:7334", -1), VeloxUserError); - EXPECT_THROW(getIPPrefix("2001:0db8:85a3:0001:0001:8a2e:0370:7334", 140), VeloxUserError); - + EXPECT_THROW( + getIPPrefix("q001:0db8:85a3:0001:0001:8a2e:0370:7334", 8), + VeloxUserError); + EXPECT_THROW( + getIPPrefix("2001:0db8:85a3:542e:0001:0001:8a2e:0370:7334", 8), + VeloxUserError); + EXPECT_THROW( + getIPPrefix("2001:0db8:85a3:0001:0001:8a2e:0370", 8), VeloxUserError); + EXPECT_THROW( + getIPPrefix("2001:0db8:85a3:0001:0001:8a2e:0370:7334", -1), + VeloxUserError); + EXPECT_THROW( + getIPPrefix("2001:0db8:85a3:0001:0001:8a2e:0370:7334", 140), + VeloxUserError); } TEST_F(IPAddressTest, castRoundTripPrefix) { - auto strings = makeFlatVector({ - "87a0:ce14:8989:44c9:826e:b4d8:73f9:1542/48", - "7cd6:bcec:1216:5c20:4b67:b1bd:173:ced/5", - "192.128.0.0/5" - }); + auto strings = makeFlatVector( + {"87a0:ce14:8989:44c9:826e:b4d8:73f9:1542/48", + "7cd6:bcec:1216:5c20:4b67:b1bd:173:ced/5", + "192.128.0.0/5"}); auto ipprefixes = evaluate("cast(c0 as ipprefix)", makeRowVector({strings})); - auto stringsCopy = evaluate("cast(c0 as varchar)", makeRowVector({ipprefixes})); - auto ipprefixesCopy = evaluate("cast(c0 as ipprefix)", makeRowVector({stringsCopy})); + auto stringsCopy = + evaluate("cast(c0 as varchar)", makeRowVector({ipprefixes})); + auto ipprefixesCopy = + evaluate("cast(c0 as ipprefix)", makeRowVector({stringsCopy})); - // assertEqualVectors are comparing the shared pointers so cannot compare + // assertEqualVectors are comparing the shared pointers so cannot compare // ipprefixes and ipprefixesCopy velox::test::assertEqualVectors(strings, stringsCopy); - } -} +} // namespace -} \ No newline at end of file +} // namespace facebook::velox::functions::prestosql \ No newline at end of file diff --git a/velox/functions/prestosql/tests/ProbabilityTest.cpp b/velox/functions/prestosql/tests/ProbabilityTest.cpp index 0c2e79d5bd984..4d01a53903a4d 100644 --- a/velox/functions/prestosql/tests/ProbabilityTest.cpp +++ b/velox/functions/prestosql/tests/ProbabilityTest.cpp @@ -496,7 +496,8 @@ TEST_F(ProbabilityTest, inverseNormalCDF) { VELOX_ASSERT_THROW(inverseNormalCDF(0, 1, kDoubleMax), "p must be 0 > p > 1"); VELOX_ASSERT_THROW( inverseNormalCDF(0, 1, kDoubleMin), - "Error in function boost::math::erf_inv(double, double): Overflow Error"); + "Error in function boost::math::erf_inv(double, double): Overflow +Error"); VELOX_ASSERT_THROW( inverseNormalCDF(0, 0, 0.1985), "standardDeviation must be > 0"); diff --git a/velox/functions/prestosql/types/IPAddressType.cpp b/velox/functions/prestosql/types/IPAddressType.cpp index 5829da8f58d51..3bcbdfd0cb026 100644 --- a/velox/functions/prestosql/types/IPAddressType.cpp +++ b/velox/functions/prestosql/types/IPAddressType.cpp @@ -43,7 +43,8 @@ class IPAddressCastOperator : public exec::CastOperator { castFromString(input, context, rows, *result); } else { VELOX_UNSUPPORTED( - "Cast from {} to IPAddress not yet supported", resultType->toString()); + "Cast from {} to IPAddress not yet supported", + resultType->toString()); } } @@ -59,7 +60,8 @@ class IPAddressCastOperator : public exec::CastOperator { castToString(input, context, rows, *result); } else { VELOX_UNSUPPORTED( - "Cast from IPAddress to {} not yet supported", resultType->toString()); + "Cast from IPAddress to {} not yet supported", + resultType->toString()); } } @@ -80,10 +82,11 @@ class IPAddressCastOperator : public exec::CastOperator { auto v6Addr = boost::asio::ip::make_address_v6(addrBytes); - if(v6Addr.is_v4_mapped()){ - auto v4Addr = boost::asio::ip::make_address_v4(boost::asio::ip::v4_mapped, v6Addr); + if (v6Addr.is_v4_mapped()) { + auto v4Addr = boost::asio::ip::make_address_v4( + boost::asio::ip::v4_mapped, v6Addr); s = boost::lexical_cast(v4Addr); - }else{ + } else { s = boost::lexical_cast(v6Addr); } @@ -104,11 +107,13 @@ class IPAddressCastOperator : public exec::CastOperator { context.applyToSelectedNoThrow(rows, [&](auto row) { const auto ipAddressString = ipAddressStrings->valueAt(row); boost::asio::ip::address_v6::bytes_type addrBytes; - auto addr = boost::asio::ip::make_address(ipAddressString); + auto addr = boost::asio::ip::make_address(ipAddressString); int128_t intAddr; - if(addr.is_v4()){ - addrBytes = boost::asio::ip::make_address_v6(boost::asio::ip::v4_mapped, addr.to_v4()).to_bytes(); - }else{ + if (addr.is_v4()) { + addrBytes = boost::asio::ip::make_address_v6( + boost::asio::ip::v4_mapped, addr.to_v4()) + .to_bytes(); + } else { addrBytes = addr.to_v6().to_bytes(); } @@ -135,7 +140,8 @@ class IPAddressTypeFactories : public CustomTypeFactories { } // namespace void registerIPAddressType() { - registerCustomType("ipaddress", std::make_unique()); + registerCustomType( + "ipaddress", std::make_unique()); } } // namespace facebook::velox diff --git a/velox/functions/prestosql/types/IPAddressType.h b/velox/functions/prestosql/types/IPAddressType.h index 33522d82db772..a299acfb800ca 100644 --- a/velox/functions/prestosql/types/IPAddressType.h +++ b/velox/functions/prestosql/types/IPAddressType.h @@ -15,14 +15,14 @@ */ #pragma once -#include "velox/expression/CastExpr.h" -#include "velox/type/SimpleFunctionApi.h" -#include "velox/type/Type.h" -#include +#include #include #include -#include #include +#include +#include "velox/expression/CastExpr.h" +#include "velox/type/SimpleFunctionApi.h" +#include "velox/type/Type.h" namespace facebook::velox { @@ -35,7 +35,8 @@ class IPAddressType : public HugeintType { public: static const std::shared_ptr& get() { - static const std::shared_ptr instance{new IPAddressType()}; + static const std::shared_ptr instance{ + new IPAddressType()}; return instance; } diff --git a/velox/functions/prestosql/types/IPPrefixType.cpp b/velox/functions/prestosql/types/IPPrefixType.cpp index 685b3050ca38d..501abe3b9d124 100644 --- a/velox/functions/prestosql/types/IPPrefixType.cpp +++ b/velox/functions/prestosql/types/IPPrefixType.cpp @@ -73,18 +73,22 @@ class IPPrefixCastOperator : public exec::CastOperator { const auto* ipaddresses = input.as>>(); context.applyToSelectedNoThrow(rows, [&](auto row) { - const auto intAddr = std::static_pointer_cast(ipaddresses->valueAt(row)); + const auto intAddr = + std::static_pointer_cast(ipaddresses->valueAt(row)); boost::asio::ip::address_v6::bytes_type addrBytes; std::string s; memcpy(&addrBytes, &intAddr->ip, 16); auto v6Addr = boost::asio::ip::make_address_v6(addrBytes); - if(v6Addr.is_v4_mapped()){ - auto v4Addr = boost::asio::ip::make_address_v4(boost::asio::ip::v4_mapped, v6Addr); - auto v4Net = boost::asio::ip::network_v4(v4Addr, (uint8_t)intAddr->prefix); + if (v6Addr.is_v4_mapped()) { + auto v4Addr = boost::asio::ip::make_address_v4( + boost::asio::ip::v4_mapped, v6Addr); + auto v4Net = + boost::asio::ip::network_v4(v4Addr, (uint8_t)intAddr->prefix); s = boost::lexical_cast(v4Net); - }else{ - auto v6Net = boost::asio::ip::network_v6(v6Addr, (uint8_t)intAddr->prefix); + } else { + auto v6Net = + boost::asio::ip::network_v6(v6Addr, (uint8_t)intAddr->prefix); s = boost::lexical_cast(v6Net); } @@ -109,18 +113,23 @@ class IPPrefixCastOperator : public exec::CastOperator { boost::asio::ip::address_v6 v6Addr; boost::asio::ip::address_v6::bytes_type addrBytes; - auto addr = boost::asio::ip::make_address(ipOnly); - IPPrefix res(0,0); - if(addr.is_v4()){ - res.prefix = (uint8_t)boost::asio::ip::make_network_v4(ipAddressString).prefix_length(); - addrBytes = boost::asio::ip::make_address_v6(boost::asio::ip::v4_mapped, addr.to_v4()).to_bytes(); - }else{ - res.prefix = (uint8_t)boost::asio::ip::make_network_v6(ipAddressString).prefix_length(); + auto addr = boost::asio::ip::make_address(ipOnly); + IPPrefix res(0, 0); + if (addr.is_v4()) { + res.prefix = (uint8_t)boost::asio::ip::make_network_v4(ipAddressString) + .prefix_length(); + addrBytes = boost::asio::ip::make_address_v6( + boost::asio::ip::v4_mapped, addr.to_v4()) + .to_bytes(); + } else { + res.prefix = (uint8_t)boost::asio::ip::make_network_v6(ipAddressString) + .prefix_length(); addrBytes = addr.to_v6().to_bytes(); } memcpy(&res.ip, &addrBytes, 16); - flatResult->set(row, std::make_shared(res.ip, (uint8_t)res.prefix)); + flatResult->set( + row, std::make_shared(res.ip, (uint8_t)res.prefix)); }); } }; @@ -139,8 +148,8 @@ class IPPrefixTypeFactories : public CustomTypeFactories { } // namespace void registerIPPrefixType() { - registerCustomType("ipprefix", std::make_unique()); + registerCustomType( + "ipprefix", std::make_unique()); } - } // namespace facebook::velox diff --git a/velox/functions/prestosql/types/IPPrefixType.h b/velox/functions/prestosql/types/IPPrefixType.h index 77bb18fbe298a..2a0f552e92448 100644 --- a/velox/functions/prestosql/types/IPPrefixType.h +++ b/velox/functions/prestosql/types/IPPrefixType.h @@ -15,15 +15,15 @@ */ #pragma once -#include "velox/expression/CastExpr.h" -#include "velox/type/SimpleFunctionApi.h" -#include "velox/type/Type.h" -#include +#include #include #include -#include #include +#include +#include "velox/expression/CastExpr.h" #include "velox/type/OpaqueCustomTypes.h" +#include "velox/type/SimpleFunctionApi.h" +#include "velox/type/Type.h" #include @@ -33,10 +33,8 @@ struct IPPrefix { int128_t ip; int8_t prefix; explicit IPPrefix(int128_t _ip, int8_t _prefix) : ip{_ip}, prefix{_prefix} {} - }; - class IPPrefixType : public OpaqueType { IPPrefixType() : OpaqueType(std::type_index(typeid(IPPrefix))) {} diff --git a/velox/functions/prestosql/types/tests/IPPrefixTypeTest.cpp b/velox/functions/prestosql/types/tests/IPPrefixTypeTest.cpp index 94066411b8270..08b003d6b13aa 100644 --- a/velox/functions/prestosql/types/tests/IPPrefixTypeTest.cpp +++ b/velox/functions/prestosql/types/tests/IPPrefixTypeTest.cpp @@ -27,7 +27,7 @@ class IPPrefixTypeTest : public testing::Test, public TypeTestBase { TEST_F(IPPrefixTypeTest, basic) { ASSERT_EQ(IPPREFIX()->name(), "IPPREFIX"); - //ASSERT_EQ(IPPREFIX()->kindName(), ""); + // ASSERT_EQ(IPPREFIX()->kindName(), ""); ASSERT_TRUE(IPPREFIX()->parameters().empty()); ASSERT_EQ(IPPREFIX()->toString(), "IPPREFIX");