From e4549b7eb2edd15112bb4425eed1936d5dd9fa1f Mon Sep 17 00:00:00 2001 From: ltrk2 <107155950+ltrk2@users.noreply.github.com> Date: Thu, 24 Nov 2022 12:53:21 -0800 Subject: [PATCH] Implement KQL UUID base64 coding functions --- .../KustoFunctions/KQLStringFunctions.cpp | 46 +++++++++---------- .../tests/KQL/gtest_KQL_StringFunctions.cpp | 8 ++-- .../02366_kql_func_string.reference | 16 ++++--- .../0_stateless/02366_kql_func_string.sql | 18 +++++--- 4 files changed, 46 insertions(+), 42 deletions(-) diff --git a/src/Parsers/Kusto/KustoFunctions/KQLStringFunctions.cpp b/src/Parsers/Kusto/KustoFunctions/KQLStringFunctions.cpp index 29ed333e60c1..16ead89c73be 100644 --- a/src/Parsers/Kusto/KustoFunctions/KQLStringFunctions.cpp +++ b/src/Parsers/Kusto/KustoFunctions/KQLStringFunctions.cpp @@ -1,14 +1,15 @@ +#include #include #include -#include #include -#include -#include -#include +#include + #include #include #include +#include + namespace DB::ErrorCodes { @@ -24,31 +25,21 @@ namespace DB bool Base64EncodeToString::convertImpl(String & out, IParser::Pos & pos) { - return directMapping(out,pos,"base64Encode"); + return directMapping(out, pos, "base64Encode"); } bool Base64EncodeFromGuid::convertImpl(String & out, IParser::Pos & pos) { - const String fn_name = getKQLFunctionName(pos); - if (fn_name.empty()) + const auto function_name = getKQLFunctionName(pos); + if (function_name.empty()) return false; - - String guid; - - ++pos; - if(pos->type == TokenType::QuotedIdentifier || pos->type == TokenType::StringLiteral) - { - --pos; - const String arg = getArgument(fn_name, pos); - guid = kqlCallToExpression("guid", {arg}, pos.max_depth); - } - else - { - guid = getConvertedArgument(fn_name, pos); - } - - out = std::format("base64Encode(toString({}))", guid); + const auto argument = getArgument(function_name, pos); + out = std::format( + "if(toTypeName({0}) not in ['UUID', 'Nullable(UUID)'], toString(throwIf(true, 'Expected guid as argument')), " + "base64Encode(UUIDStringToNum(toString({0}), 2)))", + argument, + generateUniqueIdentifier()); return true; } @@ -73,7 +64,14 @@ bool Base64DecodeToArray::convertImpl(String & out, IParser::Pos & pos) bool Base64DecodeToGuid::convertImpl(String & out, IParser::Pos & pos) { - return directMapping(out, pos, "base64Decode"); + const auto function_name = getKQLFunctionName(pos); + if (function_name.empty()) + return false; + + const auto argument = getArgument(function_name, pos); + out = std::format("toUUIDOrNull(UUIDNumToString(toFixedString(base64Decode({}), 16), 2))", argument); + + return true; } bool CountOf::convertImpl(String & out, IParser::Pos & pos) diff --git a/src/Parsers/tests/KQL/gtest_KQL_StringFunctions.cpp b/src/Parsers/tests/KQL/gtest_KQL_StringFunctions.cpp index b7ea8833dc33..5dd5b80b0441 100644 --- a/src/Parsers/tests/KQL/gtest_KQL_StringFunctions.cpp +++ b/src/Parsers/tests/KQL/gtest_KQL_StringFunctions.cpp @@ -7,12 +7,12 @@ INSTANTIATE_TEST_SUITE_P(ParserKQLQuery_String, ParserTest, ::testing::Values(std::make_shared()), ::testing::ValuesIn(std::initializer_list{ { - "print Quine = base64_encode_fromguid('ae3133f2-6e22-49ae-b06a-16e6a9b212eb')", - "SELECT base64Encode(toString(toUUIDOrNull('ae3133f2-6e22-49ae-b06a-16e6a9b212eb'))) AS Quine" + "print base64_encode_fromguid(A)", + "SELECT if(toTypeName(A) NOT IN ['UUID', 'Nullable(UUID)'], toString(throwIf(true, 'Expected guid as argument')), base64Encode(UUIDStringToNum(toString(A), 2)))" }, { - "print base64_decode_toguid('YWUzMTMzZjItNmUyMi00OWFlLWIwNmEtMTZlNmE5YjIxMmVi')", - "SELECT base64Decode('YWUzMTMzZjItNmUyMi00OWFlLWIwNmEtMTZlNmE5YjIxMmVi')" + "print base64_decode_toguid(A)", + "SELECT toUUIDOrNull(UUIDNumToString(toFixedString(base64Decode(A), 16), 2))" }, { "print base64_decode_toarray('S3VzdG8=')", diff --git a/tests/queries/0_stateless/02366_kql_func_string.reference b/tests/queries/0_stateless/02366_kql_func_string.reference index 5e09d335ac82..9bdd38ca5dba 100644 --- a/tests/queries/0_stateless/02366_kql_func_string.reference +++ b/tests/queries/0_stateless/02366_kql_func_string.reference @@ -134,12 +134,6 @@ Theodore Diaz Skilled Manual Bachelors 28 Stephanie Cox Management abcd defg Bachelors 33 Apple Skilled Manual Bachelors 28 --- Customers | project base64_encode_tostring(\'Kusto1\') | take 1 -S3VzdG8x - --- Customers | project base64_decode_tostring(\'S3VzdG8x\') | take 1 -Kusto1 - -- Customers | where isempty(LastName) Apple Skilled Manual Bachelors 28 @@ -283,11 +277,19 @@ iPhone 2 -1 -- base64_encode_fromguid() -YWUzMTMzZjItNmUyMi00OWFlLWIwNmEtMTZlNmE5YjIxMmVi +8jMxriJurkmwahbmqbIS6w== -- base64_decode_toarray() +[] [75,117,115,116,111] -- base64_decode_toguid() +10e99626-bc2b-4c75-bb3e-fe606de25700 1 +-- base64_encode_tostring + +S3VzdG8x +-- base64_decode_tostring + +Kusto1 -- parse_url() {"Scheme":"scheme","Host":"","Port":"0","Path":"/this/is/a/path","Username":"username","Password":"password","Query Parameters":{"k1":"v1","k2":"v2"},"Fragment":"fragment"} -- parse_urlquery() diff --git a/tests/queries/0_stateless/02366_kql_func_string.sql b/tests/queries/0_stateless/02366_kql_func_string.sql index 109beb391953..34fe594f37a1 100644 --- a/tests/queries/0_stateless/02366_kql_func_string.sql +++ b/tests/queries/0_stateless/02366_kql_func_string.sql @@ -115,12 +115,6 @@ print ''; print '-- Customers | where FirstName !startswith_cs \'pet\''; Customers | where FirstName !startswith_cs 'pet'| order by LastName; print ''; -print '-- Customers | project base64_encode_tostring(\'Kusto1\') | take 1'; -Customers | project base64_encode_tostring('Kusto1') | take 1; -print ''; -print '-- Customers | project base64_decode_tostring(\'S3VzdG8x\') | take 1'; -Customers | project base64_decode_tostring('S3VzdG8x') | take 1; -print ''; print '-- Customers | where isempty(LastName)'; Customers | where isempty(LastName); print ''; @@ -224,12 +218,22 @@ Customers | project indexof('abcdefg','cde') | take 1; Customers | project indexof('abcdefg','cde',2) | take 1; Customers | project indexof('abcdefg','cde',6) | take 1; print '-- base64_encode_fromguid()'; +-- print base64_encode_fromguid(guid(null)); print base64_encode_fromguid(guid('ae3133f2-6e22-49ae-b06a-16e6a9b212eb')); --- print base64_encode_fromguid("abcd1231"); exception is expected +print base64_encode_fromguid(dynamic(null)); -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO } +print base64_encode_fromguid("abcd1231"); -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO } print '-- base64_decode_toarray()'; +print base64_decode_toarray(''); print base64_decode_toarray('S3VzdG8='); print '-- base64_decode_toguid()'; +print base64_decode_toguid("JpbpECu8dUy7Pv5gbeJXAA=="); print base64_decode_toguid(base64_encode_fromguid(guid('ae3133f2-6e22-49ae-b06a-16e6a9b212eb'))) == guid('ae3133f2-6e22-49ae-b06a-16e6a9b212eb'); +print '-- base64_encode_tostring'; +print base64_encode_tostring(''); +print base64_encode_tostring('Kusto1'); +print '-- base64_decode_tostring'; +print base64_decode_tostring(''); +print base64_decode_tostring('S3VzdG8x'); print '-- parse_url()'; print parse_url('scheme://username:password@host:1234/this/is/a/path?k1=v1&k2=v2#fragment'); print '-- parse_urlquery()';