From f3c52d1f26046303246df8b04a0d9f89e3ba546c Mon Sep 17 00:00:00 2001 From: ltrk2 <107155950+ltrk2@users.noreply.github.com> Date: Mon, 19 Sep 2022 14:50:33 -0700 Subject: [PATCH] Implement KQL functional tests and fix issues --- .../KustoFunctions/IParserKQLFunction.cpp | 64 +++++++++++-------- .../Kusto/KustoFunctions/IParserKQLFunction.h | 25 +++++++- .../KustoFunctions/KQLDynamicFunctions.cpp | 54 ++++++++++++++-- .../Kusto/KustoFunctions/KQLIPFunctions.cpp | 2 +- src/Parsers/tests/KQL/gtest_KQL_Dynamic.cpp | 32 +++++----- src/Parsers/tests/KQL/gtest_KQL_IP.cpp | 4 +- .../0_stateless/02366_kql_func_dynamic.sql | 4 +- ...eference => 02366_kql_func_math.reference} | 0 ...6_kql_math.sql => 02366_kql_func_math.sql} | 0 ...reference => 02366_kql_mvexpand.reference} | 0 ...ql_mvexpend.sql => 02366_kql_mvexpand.sql} | 0 11 files changed, 129 insertions(+), 56 deletions(-) rename tests/queries/0_stateless/{02366_kql_math.reference => 02366_kql_func_math.reference} (100%) rename tests/queries/0_stateless/{02366_kql_math.sql => 02366_kql_func_math.sql} (100%) rename tests/queries/0_stateless/{02366_kql_mvexpend.reference => 02366_kql_mvexpand.reference} (100%) rename tests/queries/0_stateless/{02366_kql_mvexpend.sql => 02366_kql_mvexpand.sql} (100%) diff --git a/src/Parsers/Kusto/KustoFunctions/IParserKQLFunction.cpp b/src/Parsers/Kusto/KustoFunctions/IParserKQLFunction.cpp index e077d553d4bc..8fbbb57de2b9 100644 --- a/src/Parsers/Kusto/KustoFunctions/IParserKQLFunction.cpp +++ b/src/Parsers/Kusto/KustoFunctions/IParserKQLFunction.cpp @@ -12,11 +12,11 @@ #include #include #include +#include #include #include #include #include -#include #include @@ -73,40 +73,46 @@ bool IParserKQLFunction::convert(String & out, IParser::Pos & pos) }); } -bool IParserKQLFunction::directMapping(String & out, IParser::Pos & pos, const String & ch_fn) +bool IParserKQLFunction::directMapping( + String & out, IParser::Pos & pos, const std::string_view ch_fn, const Interval & argument_count_interval) { - std::vector arguments; - - String fn_name = getKQLFunctionName(pos); - + const auto fn_name = getKQLFunctionName(pos); if (fn_name.empty()) return false; - String res; - auto begin = pos; - ++pos; + out.append(ch_fn.data(), ch_fn.length()); + out.push_back('('); + + int argument_count = 0; + const auto begin = pos; while (!pos->isEnd() && pos->type != TokenType::PipeMark && pos->type != TokenType::Semicolon) { - String argument = getConvertedArgument(fn_name, pos); - arguments.push_back(argument); + if (pos != begin) + out.append(", "); - if (pos->type == TokenType::ClosingRoundBracket) + if (const auto argument = getOptionalArgument(fn_name, pos)) { - for (auto arg : arguments) - { - if (res.empty()) - res = ch_fn + "(" + arg; - else - res = res + ", " + arg; - } - res += ")"; + ++argument_count; + out.append(*argument); + } - out = res; + if (pos->type == TokenType::ClosingRoundBracket) + { + if (!argument_count_interval.IsWithinBounds(argument_count)) + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "{}: between {} and {} arguments are expected, but {} were provided", + fn_name, + argument_count_interval.Min(), + argument_count_interval.Max(), + argument_count); + + out.push_back(')'); return true; } - ++pos; } + out.clear(); pos = begin; return false; } @@ -174,10 +180,13 @@ String IParserKQLFunction::getConvertedArgument(const String & fn_name, IParser: std::optional IParserKQLFunction::getOptionalArgument(const String & function_name, DB::IParser::Pos & pos, const ArgumentState argument_state) { - if (const auto & type = pos->type; type != DB::TokenType::Comma && type != DB::TokenType::OpeningRoundBracket) + if (const auto type = pos->type; type != DB::TokenType::Comma && type != DB::TokenType::OpeningRoundBracket) return {}; ++pos; + if (const auto type = pos->type; type == DB::TokenType::ClosingRoundBracket || type == DB::TokenType::ClosingSquareBracket) + return {}; + if (argument_state == ArgumentState::Parsed) return getConvertedArgument(function_name, pos); @@ -187,7 +196,7 @@ IParserKQLFunction::getOptionalArgument(const String & function_name, DB::IParse "Argument extraction is not implemented for {}::{}", magic_enum::enum_type_name(), magic_enum::enum_name(argument_state)); - + String expression; std::stack scopes; while (!pos->isEnd() && (!scopes.empty() || (pos->type != DB::TokenType::Comma && pos->type != DB::TokenType::ClosingRoundBracket))) @@ -197,11 +206,12 @@ IParserKQLFunction::getOptionalArgument(const String & function_name, DB::IParse else if (isClosingBracket(token_type)) { if (scopes.empty() || determineClosingPair(scopes.top()) != token_type) - throw Exception(DB::ErrorCodes::SYNTAX_ERROR, "Unmatched token: {} when parsing {}", magic_enum::enum_name(token_type), function_name); - + throw Exception( + DB::ErrorCodes::SYNTAX_ERROR, "Unmatched token: {} when parsing {}", magic_enum::enum_name(token_type), function_name); + scopes.pop(); } - + expression.append(pos->begin, pos->end); ++pos; } diff --git a/src/Parsers/Kusto/KustoFunctions/IParserKQLFunction.h b/src/Parsers/Kusto/KustoFunctions/IParserKQLFunction.h index 45a65bd1d587..639c684bd1e1 100644 --- a/src/Parsers/Kusto/KustoFunctions/IParserKQLFunction.h +++ b/src/Parsers/Kusto/KustoFunctions/IParserKQLFunction.h @@ -7,6 +7,25 @@ namespace DB { +class Interval +{ +public: + using Representation = int; + + Interval(const Representation min_, const Representation max_) : max(max_), min(min_) { } + + Representation Max() const { return max; } + Representation Min() const { return min; } + bool IsWithinBounds(const Representation value) const { return min <= value && value <= max; } + + static constexpr auto max_bound = std::numeric_limits::max(); + static constexpr auto min_bound = std::numeric_limits::min(); + +private: + Representation max = max_bound; + Representation min = min_bound; +}; + class IParserKQLFunction { public: @@ -51,11 +70,13 @@ class IParserKQLFunction virtual bool convertImpl(String & out, IParser::Pos & pos) = 0; - static bool directMapping(String & out, IParser::Pos & pos, const String & ch_fn); + static bool directMapping( + String & out, IParser::Pos & pos, std::string_view ch_fn, const Interval & argument_count_interval = {0, Interval::max_bound}); static String generateUniqueIdentifier(); static String getArgument(const String & function_name, DB::IParser::Pos & pos, ArgumentState argument_state = ArgumentState::Parsed); static String getConvertedArgument(const String & fn_name, IParser::Pos & pos); - static std::optional getOptionalArgument(const String & function_name, DB::IParser::Pos & pos, ArgumentState argument_state = ArgumentState::Parsed); + static std::optional + getOptionalArgument(const String & function_name, DB::IParser::Pos & pos, ArgumentState argument_state = ArgumentState::Parsed); static String kqlCallToExpression(std::string_view function_name, std::initializer_list params, uint32_t max_depth); static String kqlCallToExpression(std::string_view function_name, std::span params, uint32_t max_depth); diff --git a/src/Parsers/Kusto/KustoFunctions/KQLDynamicFunctions.cpp b/src/Parsers/Kusto/KustoFunctions/KQLDynamicFunctions.cpp index f2e4f3b8d46e..b9c6da9c6704 100644 --- a/src/Parsers/Kusto/KustoFunctions/KQLDynamicFunctions.cpp +++ b/src/Parsers/Kusto/KustoFunctions/KQLDynamicFunctions.cpp @@ -21,6 +21,10 @@ String wrapInDynamic(const String & parameter) namespace DB { +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} bool ArrayConcat::convertImpl(String & out, IParser::Pos & pos) { @@ -159,8 +163,8 @@ bool ArraySlice::convertImpl(String & out, IParser::Pos & pos) const auto end = getArgument(function_name, pos); out = std::format( - "arraySlice({0}, plus(1, if({1} >= 0, {1}, toInt64(max2(-length({0}), {1})) + length({0}))) as offset_{3}, " - " plus(1, if({2} >= 0, {2}, toInt64(max2(-length({0}), {2})) + length({0}))) - offset_{3} + 1)", + "arraySlice({0}, plus(1, if({1} >= 0, {1}, arrayMax([-length({0}), {1}]) + length({0}))) as offset_{3}, " + " plus(1, if({2} >= 0, {2}, arrayMax([-length({0}), {2}]) + length({0}))) - offset_{3} + 1)", array, start, end, @@ -195,7 +199,7 @@ bool ArraySplit::convertImpl(String & out, IParser::Pos & pos) const auto indices = getArgument(function_name, pos); out = std::format( - "if(empty(arrayMap(x -> if(x >= 0, x, toInt64(max2(0, x + length({0})))), flatten([{1}])) as indices_{2}), [{0}], " + "if(empty(arrayMap(x -> if(x >= 0, x, arrayMax([0, x + length({0})::Int64])), flatten([{1}])) as indices_{2}), [{0}], " "arrayConcat([arraySlice({0}, 1, indices_{2}[1])], arrayMap(i -> arraySlice({0}, indices_{2}[i] + 1, " "if(i = length(indices_{2}), length({0})::Int64, indices_{2}[i + 1]::Int64) - indices_{2}[i]), " "range(1, length(indices_{2}) + 1))))", @@ -264,7 +268,7 @@ bool PackAll::convertImpl(String & out, IParser::Pos & pos) bool PackArray::convertImpl(String & out, IParser::Pos & pos) { - return directMapping(out, pos, "array"); + return directMapping(out, pos, "array", {1, Interval::max_bound}); } bool Repeat::convertImpl(String & out, IParser::Pos & pos) @@ -331,10 +335,48 @@ bool TreePath::convertImpl(String & out, IParser::Pos & pos) bool Zip::convertImpl(String & out, IParser::Pos & pos) { - if (!directMapping(out, pos, "arrayZip")) + const auto function_name = getKQLFunctionName(pos); + if (function_name.empty()) return false; - out = std::format("arrayMap(t -> [untuple(t)], {0})", out); + const auto arguments = std::invoke( + [&function_name, &pos] + { + std::vector result; + while (auto argument = getOptionalArgument(function_name, pos)) + result.push_back(std::move(*argument)); + + return result; + }); + + if (const auto size = arguments.size(); size < 2 || size > 16) + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Between 2 and 16 arguments are expected, but {} were provided", size); + + const auto unique_identifier = generateUniqueIdentifier(); + const auto resized_arguments = std::invoke( + [&arguments, &unique_identifier] + { + String lengths; + for (int i = 0; i < std::ssize(arguments); ++i) + { + lengths.append(i > 0 ? ", " : ""); + lengths.append(std::format( + "length(if(match(toTypeName({0}), 'Array\\(Nullable\\(.*\\)\\)'), {0}, " + "cast({0}, concat('Array(Nullable(', extract(toTypeName({0}), 'Array\\((.*)\\)'), '))'))) as arg{1}_{2})", + arguments[i], + i, + unique_identifier)); + } + + auto result = std::format("arrayResize(arg0_{1}, arrayMax([{0}]) as max_length_{1}, null)", lengths, unique_identifier); + for (int i = 1; i < std::ssize(arguments); ++i) + result.append(std::format(", arrayResize(arg{0}_{1}, max_length_{1}, null)", i, unique_identifier)); + + return result; + }); + + out = std::format("arrayMap(t -> [untuple(t)], arrayZip({0}))", resized_arguments); return true; } diff --git a/src/Parsers/Kusto/KustoFunctions/KQLIPFunctions.cpp b/src/Parsers/Kusto/KustoFunctions/KQLIPFunctions.cpp index 8328639cdc21..7baa7fc55cf7 100644 --- a/src/Parsers/Kusto/KustoFunctions/KQLIPFunctions.cpp +++ b/src/Parsers/Kusto/KustoFunctions/KQLIPFunctions.cpp @@ -152,7 +152,7 @@ bool ParseIpv4Mask::convertImpl(String & out, IParser::Pos & pos) const auto mask = getArgument(function_name, pos); out = std::format( "if(isNull(toIPv4OrNull({0}) as ip_{2}) or isNull(toUInt8OrNull(toString({1})) as mask_{2}), null, " - "toUInt32(tupleElement(IPv4CIDRToRange(assumeNotNull(ip_{2}), toUInt8(max2(0, min2(32, assumeNotNull(mask_{2}))))), 1)))", + "toUInt32(tupleElement(IPv4CIDRToRange(assumeNotNull(ip_{2}), arrayMax([0, arrayMin([32, assumeNotNull(mask_{2})])])), 1)))", ip_address, mask, generateUniqueIdentifier()); diff --git a/src/Parsers/tests/KQL/gtest_KQL_Dynamic.cpp b/src/Parsers/tests/KQL/gtest_KQL_Dynamic.cpp index 4ba35361db0c..35b34af88f84 100644 --- a/src/Parsers/tests/KQL/gtest_KQL_Dynamic.cpp +++ b/src/Parsers/tests/KQL/gtest_KQL_Dynamic.cpp @@ -49,11 +49,11 @@ INSTANTIATE_TEST_SUITE_P(ParserKQLQuery_DynamicExactMatch, ParserTest, }, { "print array_rotate_left(A, B)", - "SELECT arrayMap(x -> (A[(((x + length(A)) + (B % toInt64(length(A)))) % length(A)) + 1]), range(0, length(A)))" + "SELECT arrayMap(x -> (A[moduloOrZero((x + length(A)) + moduloOrZero(B, toInt64(length(A))), length(A)) + 1]), range(0, length(A)))" }, { "print array_rotate_right(A, B)", - "SELECT arrayMap(x -> (A[(((x + length(A)) + ((-1 * B) % toInt64(length(A)))) % length(A)) + 1]), range(0, length(A)))" + "SELECT arrayMap(x -> (A[moduloOrZero((x + length(A)) + moduloOrZero(-1 * B, toInt64(length(A))), length(A)) + 1]), range(0, length(A)))" }, { "print output = array_sum(dynamic([2, 5, 3]))", @@ -102,14 +102,6 @@ INSTANTIATE_TEST_SUITE_P(ParserKQLQuery_DynamicExactMatch, ParserTest, { "print set_union(A, B, C)", "SELECT arrayDistinct(arrayConcat(A, B, C))" - }, - { - "print zip(A, B)", - "SELECT arrayMap(t -> [untuple(t)], arrayZip(A, B))" - }, - { - "print zip(A, B, C)", - "SELECT arrayMap(t -> [untuple(t)], arrayZip(A, B, C))" } }))); @@ -119,26 +111,34 @@ INSTANTIATE_TEST_SUITE_P(ParserKQLQuery_DynamicRegex, ParserRegexTest, ::testing::ValuesIn(std::initializer_list{ { "print array_shift_left(A, B)", - "SELECT arrayResize\\(if\\(B > 0, arraySlice\\(A, B \\+ 1\\), arrayConcat\\(arrayWithConstant\\(abs\\(B\\), fill_value_\\d+\\), A\\)\\), length\\(A\\), ifNull\\(NULL, if\\(toTypeName\\(A\\) = 'Array\\(String\\)', defaultValueOfArgumentType\\(A\\[1\\]\\), NULL\\)\\) AS fill_value_\\d+\\)" + R"(SELECT arrayResize\(if\(B > 0, arraySlice\(A, B \+ 1\), arrayConcat\(arrayWithConstant\(abs\(B\), fill_value_\d+\), A\)\), length\(A\), if\(\(NULL IS NULL\) AND \(\(extract\(toTypeName\(A\), 'Array\\\\\(\(\.\*\)\\\\\)*'\) AS element_type_\d+\) = 'String'\), defaultValueOfTypeName\(if\(element_type_\d+ = 'Nothing', 'Nullable\(Nothing\)', element_type_\d+\)\), NULL\) AS fill_value_\d+\))" }, { "print array_shift_left(A, B, C)", - "SELECT arrayResize\\(if\\(B > 0, arraySlice\\(A, B \\+ 1\\), arrayConcat\\(arrayWithConstant\\(abs\\(B\\), fill_value_\\d+\\), A\\)\\), length\\(A\\), ifNull\\(C, if\\(toTypeName\\(A\\) = 'Array\\(String\\)', defaultValueOfArgumentType\\(A\\[1\\]\\), NULL\\)\\) AS fill_value_\\d+\\)" + R"(SELECT arrayResize\(if\(B > 0, arraySlice\(A, B \+ 1\), arrayConcat\(arrayWithConstant\(abs\(B\), fill_value_\d+\), A\)\), length\(A\), if\(\(C IS NULL\) AND \(\(extract\(toTypeName\(A\), 'Array\\\\\(\(\.\*\)\\\\\)'\) AS element_type_\d+\) = 'String'\), defaultValueOfTypeName\(if\(element_type_\d+ = 'Nothing', 'Nullable\(Nothing\)', element_type_\d+\)\), C\) AS fill_value_\d+\))" }, { "print array_shift_right(A, B)", - "SELECT arrayResize\\(if\\(\\(-1 \\* B\\) > 0, arraySlice\\(A, \\(-1 \\* B\\) \\+ 1\\), arrayConcat\\(arrayWithConstant\\(abs\\(-1 \\* B\\), fill_value_\\d+\\), A\\)\\), length\\(A\\), ifNull\\(NULL, if\\(toTypeName\\(A\\) = 'Array\\(String\\)', defaultValueOfArgumentType\\(A\\[1\\]\\), NULL\\)\\) AS fill_value_\\d+\\)" + R"(SELECT arrayResize\(if\(\(-1 \* B\) > 0, arraySlice\(A, \(-1 \* B\) \+ 1\), arrayConcat\(arrayWithConstant\(abs\(-1 \* B\), fill_value_\d+\), A\)\), length\(A\), if\(\(NULL IS NULL\) AND \(\(extract\(toTypeName\(A\), 'Array\\\\\(\(\.\*\)\\\\\)'\) AS element_type_\d+\) = 'String'\), defaultValueOfTypeName\(if\(element_type_\d+ = 'Nothing', 'Nullable\(Nothing\)', element_type_\d+\)\), NULL\) AS fill_value_\d+\))" }, { "print array_shift_right(A, B, C)", - "SELECT arrayResize\\(if\\(\\(-1 \\* B\\) > 0, arraySlice\\(A, \\(-1 \\* B\\) \\+ 1\\), arrayConcat\\(arrayWithConstant\\(abs\\(-1 \\* B\\), fill_value_\\d+\\), A\\)\\), length\\(A\\), ifNull\\(C, if\\(toTypeName\\(A\\) = 'Array\\(String\\)', defaultValueOfArgumentType\\(A\\[1\\]\\), NULL\\)\\) AS fill_value_\\d+\\)" + R"(SELECT arrayResize\(if\(\(-1 \* B\) > 0, arraySlice\(A, \(-1 \* B\) \+ 1\), arrayConcat\(arrayWithConstant\(abs\(-1 \* B\), fill_value_\d+\), A\)\), length\(A\), if\(\(C IS NULL\) AND \(\(extract\(toTypeName\(A\), 'Array\\\\\(\(\.\*\)\\\\\)'\) AS element_type_\d+\) = 'String'\), defaultValueOfTypeName\(if\(element_type_\d+ = 'Nothing', 'Nullable\(Nothing\)', element_type_\d+\)\), C\) AS fill_value_\d+\))" }, { "print array_slice(A, B, C)", - "SELECT arraySlice\\(A, 1 \\+ if\\(B >= 0, B, toInt64\\(max2\\(-length\\(A\\), B\\)\\) \\+ length\\(A\\)\\) AS offset_\\d+, \\(\\(1 \\+ if\\(C >= 0, C, toInt64\\(max2\\(-length\\(A\\), C\\)\\) \\+ length\\(A\\)\\)\\) - offset_\\d+\\) \\+ 1\\)" + R"(SELECT arraySlice\(A, 1 \+ if\(B >= 0, B, arrayMax\(\[-length\(A\), B\]\) \+ length\(A\)\) AS offset_\d+, \(\(1 \+ if\(C >= 0, C, arrayMax\(\[-length\(A\), C\]\) \+ length\(A\)\)\) - offset_\d+\) \+ 1\))" }, { "print array_split(A, B)", - "SELECT if\\(empty\\(arrayMap\\(x -> if\\(x >= 0, x, toInt64\\(max2\\(0, x \\+ length\\(A\\)\\)\\)\\), flatten\\(\\[B\\]\\)\\) AS indices_\\d+\\), \\[A\\], arrayConcat\\(\\[arraySlice\\(A, 1, indices_\\d+\\[1\\]\\)\\], arrayMap\\(i -> arraySlice\\(A, \\(indices_\\d+\\[i\\]\\) \\+ 1, if\\(i = length\\(indices_\\d+\\), CAST\\(length\\(A\\), 'Int64'\\), CAST\\(indices_\\d+\\[i \\+ 1\\], 'Int64'\\)\\) - \\(indices_\\d+\\[i\\]\\)\\), range\\(1, length\\(indices_\\d+\\) \\+ 1\\)\\)\\)\\)" + R"(SELECT if\(empty\(arrayMap\(x -> if\(x >= 0, x, arrayMax\(\[0, x \+ CAST\(length\(A\), 'Int\d+'\)\]\)\), flatten\(\[B\]\)\) AS indices_\d+\), \[A\], arrayConcat\(\[arraySlice\(A, 1, indices_\d+\[1\]\)\], arrayMap\(i -> arraySlice\(A, \(indices_\d+\[i\]\) \+ 1, if\(i = length\(indices_\d+\), CAST\(length\(A\), 'Int\d+'\), CAST\(indices_\d+\[i \+ 1\], 'Int\d+'\)\) - \(indices_\d+\[i\]\)\), range\(1, length\(indices_\d+\) \+ 1\)\)\)\))" + }, + { + "print zip(A, B)", + R"(SELECT arrayMap\(t -> \[untuple\(t\)\], arrayZip\(arrayResize\(arg0_\d+, arrayMax\(\[length\(if\(match\(toTypeName\(A\), 'Array\\\\\(Nullable\\\\\(\.\*\\\\\)\\\\\)'\), A, CAST\(A, concat\('Array\(Nullable\(', extract\(toTypeName\(A\), 'Array\\\\\(\(\.\*\)\\\\\)'\), '\)\)'\)\)\) AS arg0_\d+\), length\(if\(match\(toTypeName\(B\), 'Array\\\\\(Nullable\\\\\(\.\*\\\\\)\\\\\)'\), B, CAST\(B, concat\('Array\(Nullable\(', extract\(toTypeName\(B\), 'Array\\\\\(\(\.\*\)\\\\\)'\), '\)\)'\)\)\) AS arg1_\d+\)\]\) AS max_length_\d+, NULL\), arrayResize\(arg1_\d+, max_length_\d+, NULL\)\)\))" + }, + { + "print zip(A, B, C)", + R"(SELECT arrayMap\(t -> \[untuple\(t\)\], arrayZip\(arrayResize\(arg0_\d+, arrayMax\(\[length\(if\(match\(toTypeName\(A\), 'Array\\\\\(Nullable\\\\\(\.\*\\\\\)\\\\\)'\), A, CAST\(A, concat\('Array\(Nullable\(', extract\(toTypeName\(A\), 'Array\\\\\(\(\.\*\)\\\\\)'\), '\)\)'\)\)\) AS arg0_\d+\), length\(if\(match\(toTypeName\(B\), 'Array\\\\\(Nullable\\\\\(\.\*\\\\\)\\\\\)'\), B, CAST\(B, concat\('Array\(Nullable\(', extract\(toTypeName\(B\), 'Array\\\\\(\(\.\*\)\\\\\)'\), '\)\)'\)\)\) AS arg1_\d+\), length\(if\(match\(toTypeName\(C\), 'Array\\\\\(Nullable\\\\\(\.\*\\\\\)\\\\\)'\), C, CAST\(C, concat\('Array\(Nullable\(', extract\(toTypeName\(C\), 'Array\\\\\(\(\.\*\)\\\\\)'\), '\)\)'\)\)\) AS arg2_\d+\)\]\) AS max_length_\d+, NULL\), arrayResize\(arg1_\d+, max_length_\d+, NULL\), arrayResize\(arg2_\d+, max_length_\d+, NULL\)\)\))" } }))); diff --git a/src/Parsers/tests/KQL/gtest_KQL_IP.cpp b/src/Parsers/tests/KQL/gtest_KQL_IP.cpp index 147c94b07168..f05f420fd04a 100644 --- a/src/Parsers/tests/KQL/gtest_KQL_IP.cpp +++ b/src/Parsers/tests/KQL/gtest_KQL_IP.cpp @@ -72,7 +72,7 @@ INSTANTIATE_TEST_SUITE_P(ParserKQLQuery_IP, ParserRegexTest, }, { "print parse_ipv4_mask(A, B)", - "SELECT if\\(\\(\\(toIPv4OrNull\\(A\\) AS ip_\\d+\\) IS NULL\\) OR \\(\\(toUInt8OrNull\\(toString\\(B\\)\\) AS mask_\\d+\\) IS NULL\\), NULL, toUInt32\\(IPv4CIDRToRange\\(assumeNotNull\\(ip_\\d+\\), toUInt8\\(max2\\(0, min2\\(32, assumeNotNull\\(mask_\\d+\\)\\)\\)\\)\\).1\\)\\)" + "SELECT if\\(\\(\\(toIPv4OrNull\\(A\\) AS ip_\\d+\\) IS NULL\\) OR \\(\\(toUInt8OrNull\\(toString\\(B\\)\\) AS mask_\\d+\\) IS NULL\\), NULL, toUInt32\\(IPv4CIDRToRange\\(assumeNotNull\\(ip_\\d+\\), arrayMax\\(\\[0, arrayMin\\(\\[32, assumeNotNull\\(mask_\\d+\\)\\]\\)\\]\\)\\).1\\)\\)" }, { "print parse_ipv6(A)", @@ -80,6 +80,6 @@ INSTANTIATE_TEST_SUITE_P(ParserKQLQuery_IP, ParserRegexTest, }, { "print parse_ipv6_mask(A, B)", - "SELECT if\\(\\(if\\(\\(\\(toIPv4OrNull\\(A\\) AS ip_\\d+\\) IS NULL\\) OR \\(\\(toUInt8OrNull\\(toString\\(B\\)\\) AS mask_\\d+\\) IS NULL\\), NULL, toUInt32\\(IPv4CIDRToRange\\(assumeNotNull\\(ip_\\d+\\), toUInt8\\(max2\\(0, min2\\(32, assumeNotNull\\(mask_\\d+\\)\\)\\)\\)\\).1\\)\\) AS ipv4_\\d+\\) IS NULL, if\\(\\(length\\(splitByChar\\('/', concat\\(ifNull\\(toString\\(if\\(\\(length\\(splitByChar\\('/', A\\) AS tokens_\\d+\\) > 2\\) OR \\(\\(IPv6StringToNumOrNull\\(tokens_\\d+\\[1\\]\\) AS ip_\\d+\\) IS NULL\\) OR \\(\\(length\\(tokens_\\d+\\) = 2\\) AND \\(\\(toUInt8OrNull\\(tokens_\\d+\\[-1\\]\\) AS mask_\\d+\\) IS NULL\\)\\), NULL, arrayStringConcat\\(flatten\\(extractAllGroups\\(lower\\(hex\\(IPv6CIDRToRange\\(assumeNotNull\\(ip_\\d+\\), toUInt8\\(ifNull\\(mask_\\d+ \\+ if\\(isIPv4String\\(tokens_\\d+\\[1\\]\\), 96, 0\\), 128\\)\\)\\).1\\)\\), '\\(\\[\\\\\\\\da-f\\]\\{4\\}\\)'\\)\\), ':'\\)\\)\\), ''\\), '/', ifNull\\(toString\\(B\\), ''\\)\\)\\) AS tokens_\\d+\\) > 2\\) OR \\(\\(IPv6StringToNumOrNull\\(tokens_\\d+\\[1\\]\\) AS ip_\\d+\\) IS NULL\\) OR \\(\\(length\\(tokens_\\d+\\) = 2\\) AND \\(\\(toUInt8OrNull\\(tokens_\\d+\\[-1\\]\\) AS mask_\\d+\\) IS NULL\\)\\), NULL, arrayStringConcat\\(flatten\\(extractAllGroups\\(lower\\(hex\\(IPv6CIDRToRange\\(assumeNotNull\\(ip_\\d+\\), toUInt8\\(ifNull\\(mask_\\d+ \\+ if\\(isIPv4String\\(tokens_\\d+\\[1\\]\\), 96, 0\\), 128\\)\\)\\).1\\)\\), '\\(\\[\\\\\\\\da-f\\]\\{4\\}\\)'\\)\\), ':'\\)\\), if\\(\\(length\\(splitByChar\\('/', ifNull\\(if\\(\\(\\(\\(toUInt32OrNull\\(toString\\(ipv4_\\d+\\)\\) AS param_as_uint32_\\d+\\) IS NOT NULL\\) AND \\(toTypeName\\(ipv4_\\d+\\) = 'String'\\)\\) OR \\(32 < 0\\) OR \\(\\(ifNull\\(param_as_uint32_\\d+, multiIf\\(length\\(splitByChar\\('/', ifNull\\(toString\\(ipv4_\\d+\\), ''\\)\\) AS tokens_\\d+\\) = 1, IPv4StringToNumOrNull\\(tokens_\\d+\\[1\\]\\) AS ip_\\d+, \\(length\\(tokens_\\d+\\) = 2\\) AND \\(ip_\\d+ IS NOT NULL\\) AND \\(\\(toUInt8OrNull\\(tokens_\\d+\\[-1\\]\\) AS mask_\\d+\\) IS NOT NULL\\), IPv4CIDRToRange\\(assumeNotNull\\(ip_\\d+\\), assumeNotNull\\(mask_\\d+\\)\\).1, NULL\\)\\) AS ip_as_number_\\d+\\) IS NULL\\), NULL, IPv4NumToString\\(bitAnd\\(ip_as_number_\\d+, bitNot\\(toUInt32\\(intExp2\\(32 - 32\\) - 1\\)\\)\\)\\)\\), ''\\)\\) AS tokens_\\d+\\) > 2\\) OR \\(\\(IPv6StringToNumOrNull\\(tokens_\\d+\\[1\\]\\) AS ip_\\d+\\) IS NULL\\) OR \\(\\(length\\(tokens_\\d+\\) = 2\\) AND \\(\\(toUInt8OrNull\\(tokens_\\d+\\[-1\\]\\) AS mask_\\d+\\) IS NULL\\)\\), NULL, arrayStringConcat\\(flatten\\(extractAllGroups\\(lower\\(hex\\(IPv6CIDRToRange\\(assumeNotNull\\(ip_\\d+\\), toUInt8\\(ifNull\\(mask_\\d+ \\+ if\\(isIPv4String\\(tokens_\\d+\\[1\\]\\), 96, 0\\), 128\\)\\)\\).1\\)\\), '\\(\\[\\\\\\\\da-f\\]\\{4\\}\\)'\\)\\), ':'\\)\\)\\)" + "SELECT if\\(\\(if\\(\\(\\(toIPv4OrNull\\(A\\) AS ip_\\d+\\) IS NULL\\) OR \\(\\(toUInt8OrNull\\(toString\\(B\\)\\) AS mask_\\d+\\) IS NULL\\), NULL, toUInt32\\(IPv4CIDRToRange\\(assumeNotNull\\(ip_\\d+\\), arrayMax\\(\\[0, arrayMin\\(\\[32, assumeNotNull\\(mask_\\d+\\)\\]\\)\\]\\)\\).1\\)\\) AS ipv4_\\d+\\) IS NULL, if\\(\\(length\\(splitByChar\\('/', concat\\(ifNull\\(toString\\(if\\(\\(length\\(splitByChar\\('/', A\\) AS tokens_\\d+\\) > 2\\) OR \\(\\(IPv6StringToNumOrNull\\(tokens_\\d+\\[1\\]\\) AS ip_\\d+\\) IS NULL\\) OR \\(\\(length\\(tokens_\\d+\\) = 2\\) AND \\(\\(toUInt8OrNull\\(tokens_\\d+\\[-1\\]\\) AS mask_\\d+\\) IS NULL\\)\\), NULL, arrayStringConcat\\(flatten\\(extractAllGroups\\(lower\\(hex\\(IPv6CIDRToRange\\(assumeNotNull\\(ip_\\d+\\), toUInt8\\(ifNull\\(mask_\\d+ \\+ if\\(isIPv4String\\(tokens_\\d+\\[1\\]\\), 96, 0\\), 128\\)\\)\\).1\\)\\), '\\(\\[\\\\\\\\da-f\\]\\{4\\}\\)'\\)\\), ':'\\)\\)\\), ''\\), '/', ifNull\\(toString\\(B\\), ''\\)\\)\\) AS tokens_\\d+\\) > 2\\) OR \\(\\(IPv6StringToNumOrNull\\(tokens_\\d+\\[1\\]\\) AS ip_\\d+\\) IS NULL\\) OR \\(\\(length\\(tokens_\\d+\\) = 2\\) AND \\(\\(toUInt8OrNull\\(tokens_\\d+\\[-1\\]\\) AS mask_\\d+\\) IS NULL\\)\\), NULL, arrayStringConcat\\(flatten\\(extractAllGroups\\(lower\\(hex\\(IPv6CIDRToRange\\(assumeNotNull\\(ip_\\d+\\), toUInt8\\(ifNull\\(mask_\\d+ \\+ if\\(isIPv4String\\(tokens_\\d+\\[1\\]\\), 96, 0\\), 128\\)\\)\\).1\\)\\), '\\(\\[\\\\\\\\da-f\\]\\{4\\}\\)'\\)\\), ':'\\)\\), if\\(\\(length\\(splitByChar\\('/', ifNull\\(if\\(\\(\\(\\(toUInt32OrNull\\(toString\\(ipv4_\\d+\\)\\) AS param_as_uint32_\\d+\\) IS NOT NULL\\) AND \\(toTypeName\\(ipv4_\\d+\\) = 'String'\\)\\) OR \\(32 < 0\\) OR \\(\\(ifNull\\(param_as_uint32_\\d+, multiIf\\(length\\(splitByChar\\('/', ifNull\\(toString\\(ipv4_\\d+\\), ''\\)\\) AS tokens_\\d+\\) = 1, IPv4StringToNumOrNull\\(tokens_\\d+\\[1\\]\\) AS ip_\\d+, \\(length\\(tokens_\\d+\\) = 2\\) AND \\(ip_\\d+ IS NOT NULL\\) AND \\(\\(toUInt8OrNull\\(tokens_\\d+\\[-1\\]\\) AS mask_\\d+\\) IS NOT NULL\\), IPv4CIDRToRange\\(assumeNotNull\\(ip_\\d+\\), assumeNotNull\\(mask_\\d+\\)\\).1, NULL\\)\\) AS ip_as_number_\\d+\\) IS NULL\\), NULL, IPv4NumToString\\(bitAnd\\(ip_as_number_\\d+, bitNot\\(toUInt32\\(intExp2\\(32 - 32\\) - 1\\)\\)\\)\\)\\), ''\\)\\) AS tokens_\\d+\\) > 2\\) OR \\(\\(IPv6StringToNumOrNull\\(tokens_\\d+\\[1\\]\\) AS ip_\\d+\\) IS NULL\\) OR \\(\\(length\\(tokens_\\d+\\) = 2\\) AND \\(\\(toUInt8OrNull\\(tokens_\\d+\\[-1\\]\\) AS mask_\\d+\\) IS NULL\\)\\), NULL, arrayStringConcat\\(flatten\\(extractAllGroups\\(lower\\(hex\\(IPv6CIDRToRange\\(assumeNotNull\\(ip_\\d+\\), toUInt8\\(ifNull\\(mask_\\d+ \\+ if\\(isIPv4String\\(tokens_\\d+\\[1\\]\\), 96, 0\\), 128\\)\\)\\).1\\)\\), '\\(\\[\\\\\\\\da-f\\]\\{4\\}\\)'\\)\\), ':'\\)\\)\\)" } }))); diff --git a/tests/queries/0_stateless/02366_kql_func_dynamic.sql b/tests/queries/0_stateless/02366_kql_func_dynamic.sql index 333e182a6a67..931e047421fd 100644 --- a/tests/queries/0_stateless/02366_kql_func_dynamic.sql +++ b/tests/queries/0_stateless/02366_kql_func_dynamic.sql @@ -99,8 +99,8 @@ print jaccard_index(dynamic([1, 2, 3]), dynamic([4, 5, 6, 7])); print jaccard_index(dynamic(['a', 's', 'd']), dynamic(['f', 'd', 's', 'a'])); print jaccard_index(dynamic(['Chewbacca', 'Darth Vader', 'Han Solo']), dynamic(['Darth Sidious', 'Darth Vader'])); print '-- pack_array()'; -print pack_array(); -- { serverError } -print x = 1, y = x * 2, z = y * 2, pack_array(x,y,z); +print pack_array(); -- { clientError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +print x = 1 | extend y = x * 2 | extend z = y * 2 | extend pack_array(x,y,z); print pack_array(strcat('a', 'b'), format_ipv4(42), tostring(4.2)); print '-- repeat()'; print repeat(1, 0); diff --git a/tests/queries/0_stateless/02366_kql_math.reference b/tests/queries/0_stateless/02366_kql_func_math.reference similarity index 100% rename from tests/queries/0_stateless/02366_kql_math.reference rename to tests/queries/0_stateless/02366_kql_func_math.reference diff --git a/tests/queries/0_stateless/02366_kql_math.sql b/tests/queries/0_stateless/02366_kql_func_math.sql similarity index 100% rename from tests/queries/0_stateless/02366_kql_math.sql rename to tests/queries/0_stateless/02366_kql_func_math.sql diff --git a/tests/queries/0_stateless/02366_kql_mvexpend.reference b/tests/queries/0_stateless/02366_kql_mvexpand.reference similarity index 100% rename from tests/queries/0_stateless/02366_kql_mvexpend.reference rename to tests/queries/0_stateless/02366_kql_mvexpand.reference diff --git a/tests/queries/0_stateless/02366_kql_mvexpend.sql b/tests/queries/0_stateless/02366_kql_mvexpand.sql similarity index 100% rename from tests/queries/0_stateless/02366_kql_mvexpend.sql rename to tests/queries/0_stateless/02366_kql_mvexpand.sql