Implement KQL functional tests and fix bugs that came up

src/Parsers/Kusto/KustoFunctions/KQLDynamicFunctions.cpp

@@ -86,7 +86,10 @@ bool ArrayRotateLeft::convertImpl(String & out, IParser::Pos & pos)
     const auto array = getArgument(function_name, pos);
     const auto count = getArgument(function_name, pos);
     out = std::format(
-        "arrayMap(x -> {0}[(x + length({0}) + {1} % toInt64(length({0}))) % length({0}) + 1], range(0, length({0})))", array, count);
+        "arrayMap(x -> {0}[moduloOrZero(x + length({0}) + moduloOrZero({1}, toInt64(length({0}))), length({0})) + 1], "
+        "range(0, length({0})))",
+        array,
+        count);
 
     return true;
 }
@@ -115,7 +118,8 @@ bool ArrayShiftLeft::convertImpl(String & out, IParser::Pos & pos)
     const auto fill = getOptionalArgument(function_name, pos);
     out = std::format(
         "arrayResize(if({1} > 0, arraySlice({0}, {1} + 1), arrayConcat(arrayWithConstant(abs({1}), fill_value_{3}), {0})), "
-        "length({0}), ifNull({2}, if(toTypeName({0}) = 'Array(String)', defaultValueOfArgumentType({0}[1]), null)) as fill_value_{3})",
+        "length({0}), if(isNull({2}) and (extract(toTypeName({0}), 'Array\\((.*)\\)') as element_type_{3}) = 'String', "
+        "defaultValueOfTypeName(if(element_type_{3} = 'Nothing', 'Nullable(Nothing)', element_type_{3})), {2}) as fill_value_{3})",
         array,
         count,
         fill ? *fill : "null",

src/Parsers/Kusto/KustoFunctions/KQLMathematicalFunctions.cpp

@@ -1,9 +1,18 @@
 #include "KQLMathematicalFunctions.h"
 
+#include <format>
+
 namespace DB
 {
 bool IsNan::convertImpl(String & out, IParser::Pos & pos)
 {
-    return directMapping(out, pos, "isNaN");
+    const auto function_name = getKQLFunctionName(pos);
+    if (function_name.empty())
+        return false;
+
+    const auto argument = getArgument(function_name, pos);
+    out = std::format("if(toTypeName({0}) = 'Float64', isNaN({0}), throwIf(true, 'Expected argument of data type real'))", argument);
+
+    return true;
 }
 }

tests/queries/0_stateless/02366_kql_datatype.reference

@@ -47,9 +47,12 @@ inf
 \N
 123.345
 -- dynamic
+\N
+1
+86400
 [1,2,3]
+[[1],[2],[3]]
 ['a','b','c']
-\N
 -- cast functions
 true
 1

tests/queries/0_stateless/02366_kql_datatype.sql

@@ -53,11 +53,13 @@ print bool(null), int(null), long(null), real(null), double(null);
 print '-- decimal';
 print decimal(null);
 print decimal(123.345);
-print '-- dynamic'; -- only support 1D array at the moment
-print dynamic([1,2,3]);
-print dynamic(['a', 'b', 'c']);
+print '-- dynamic'; -- no support for mixed types and bags for now
 print dynamic(null);
-
+print dynamic(1);
+print dynamic(timespan(1d));
+print dynamic([1,2,3]);
+print dynamic([[1], [2], [3]]);
+print dynamic(['a', "b", 'c']);
 print '-- cast functions'
 print '--tobool("true")'; -- == true
 print tobool('true'); -- == true

tests/queries/0_stateless/02366_kql_func_dynamic.reference

@@ -14,9 +14,51 @@
 [NULL,NULL,NULL]
 -- array_concat()
 [1,2,3,4,5,6]
+-- array_reverse()
+[]
+[1]
+[4,3,2,1]
+['example','an','is','this']
+-- array_rotate_left()
+[]
+[]
+[]
+[3,4,5,1,2]
+[1,2,3,4,5]
+[3,4,5,1,2]
+[4,5,1,2,3]
+[1,2,3,4,5]
+[4,5,1,2,3]
+-- array_rotate_right()
+[]
+[]
+[]
+[4,5,1,2,3]
+[1,2,3,4,5]
+[4,5,1,2,3]
+[3,4,5,1,2]
+[1,2,3,4,5]
+[3,4,5,1,2]
+-- array_shift_left()
+[]
+[]
+[]
+[3,4,5,NULL,NULL]
+[NULL,NULL,1,2,3]
+[3,4,5,-1,-1]
+['c','','']
+-- array_shift_right()
+[]
+[]
+[]
+[3,4,5,NULL,NULL]
+[NULL,NULL,1,2,3]
+[3,4,5,-1,-1]
+['c','','']
 -- array_slice()
 [2,3]
 [3,4]
+-- array_split()
 [[1],[2,3],[4,5]]
 [[1,2],[3,4,5]]
 [[1],[2,3],[4,5]]
@@ -40,9 +82,68 @@
 ['c','b','a']
 [['r','q','p'],['world','clickhouse','hello']]
 [[NULL,'d','c','c','a']]
+[[NULL,'d','c','c','a']]
 [[NULL,NULL,NULL]]
 [3,2,1,NULL,NULL]
 ['d','c','b','e','a']
 [['Ringo','Paul','John','George']]
 [['yellow','green','blue',NULL,NULL]]
 [[NULL,NULL,'yellow','green','blue']]
+-- jaccard_index()
+0.75
+0
+0
+nan
+0
+0.75
+0.25
+-- pack_array()
+1	2	4	[1,2,4]
+['ab','0.0.0.42','4.2']
+-- repeat()
+[]
+[1,1,1]
+['asd','asd','asd']
+[86400,86400,86400]
+[true,true,true]
+-- set_difference()
+[]
+[]
+[]
+[]
+[4,5,6]
+[4]
+[1,3]
+[1,2,3]
+['d','s']
+['Chewbacca','Han Solo']
+-- set_has_element()
+0
+1
+0
+1
+0
+-- set_intersect()
+[]
+[1,2,3]
+[1,2,3]
+[]
+[5]
+[]
+['a']
+['Darth Vader']
+-- set_union()
+[]
+[1,2,3]
+[1,2,3,4,5,6]
+[1,2,3,4]
+[1,2,3,4,5]
+[1,2,3]
+['a','d','f','s']
+['Chewbacca','Darth Sidious','Darth Vader','Han Solo']
+-- zip()
+[]
+[[1,2],[3,4],[5,6]]
+[['Darth','Vader','has a suit'],['Master','Yoda','doesn\'t have a suit']]
+[[1,10],[2,20],[3,NULL]]
+[[NULL,1],[NULL,2],[NULL,3]]

tests/queries/0_stateless/02366_kql_func_dynamic.sql

@@ -15,9 +15,50 @@ print array_iif(dynamic([true,false,true]), dynamic([1,2]), dynamic([4,5,6]));
 print array_iif(dynamic(['a','b','c']), dynamic([1,2,3]), dynamic([4,5,6]));
 print '-- array_concat()';
 print array_concat(dynamic([1,2,3]),dynamic([4,5,6]));
+print '-- array_reverse()';
+print array_reverse(dynamic([]));
+print array_reverse(dynamic([1]));
+print array_reverse(dynamic([1,2,3,4]));
+print array_reverse(dynamic(["this", "is", "an", "example"]));
+print '-- array_rotate_left()';
+print array_rotate_left(dynamic([]), 0);
+print array_rotate_left(dynamic([]), 500);
+print array_rotate_left(dynamic([]), -500);
+print array_rotate_left(dynamic([1,2,3,4,5]), 2);
+print array_rotate_left(dynamic([1,2,3,4,5]), 5);
+print array_rotate_left(dynamic([1,2,3,4,5]), 7);
+print array_rotate_left(dynamic([1,2,3,4,5]), -2);
+print array_rotate_left(dynamic([1,2,3,4,5]), -5);
+print array_rotate_left(dynamic([1,2,3,4,5]), -7);
+print '-- array_rotate_right()';
+print array_rotate_right(dynamic([]), 0);
+print array_rotate_right(dynamic([]), 500);
+print array_rotate_right(dynamic([]), -500);
+print array_rotate_right(dynamic([1,2,3,4,5]), 2);
+print array_rotate_right(dynamic([1,2,3,4,5]), 5);
+print array_rotate_right(dynamic([1,2,3,4,5]), 7);
+print array_rotate_right(dynamic([1,2,3,4,5]), -2);
+print array_rotate_right(dynamic([1,2,3,4,5]), -5);
+print array_rotate_right(dynamic([1,2,3,4,5]), -7);
+print '-- array_shift_left()';
+print array_shift_left(dynamic([]), 0);
+print array_shift_left(dynamic([]), 555);
+print array_shift_left(dynamic([]), -555);
+print array_shift_left(dynamic([1,2,3,4,5]), 2);
+print array_shift_left(dynamic([1,2,3,4,5]), -2);
+print array_shift_left(dynamic([1,2,3,4,5]), 2, -1);
+print array_shift_left(dynamic(['a', 'b', 'c']), 2);
+print '-- array_shift_right()';
+print array_shift_left(dynamic([]), 0);
+print array_shift_left(dynamic([]), 555);
+print array_shift_left(dynamic([]), -555);
+print array_shift_right(dynamic([1,2,3,4,5]), -2);
+print array_shift_right(dynamic([1,2,3,4,5]), 2);
+print array_shift_right(dynamic([1,2,3,4,5]), -2, -1);
+print array_shift_right(dynamic(['a', 'b', 'c']), -2);
 print '-- array_slice()';
 print array_slice(dynamic([1,2,3]), 1, 2);
-print array_slice(dynamic([1,2,3,4,5]), -3, -2)
+print array_slice(dynamic([1,2,3,4,5]), -3, -2);
 print '-- array_split()';
 print array_split(dynamic([1,2,3,4,5]), dynamic([1,-2]));
 print array_split(dynamic([1,2,3,4,5]), 2);
@@ -40,7 +81,7 @@ print '-- array_sort_desc()';
 print t = array_sort_desc(dynamic([null, 'd', 'a', 'c', 'c']));
 print t = array_sort_desc(dynamic([4, 1, 3, 2]));
 print t = array_sort_desc(dynamic(['b', 'a', 'c']), dynamic(['q', 'p', 'r']))[1];
-print array_sort_desc(dynamic(['q', 'p', 'r']), dynamic(['clickhouse','hello', 'world']))
+print array_sort_desc(dynamic(['q', 'p', 'r']), dynamic(['clickhouse','hello', 'world']));
 print t = array_sort_desc( dynamic(['d', null, 'a', 'c', 'c']) , false);
 print t = array_sort_desc( dynamic(['d', null, 'a', 'c', 'c']) , 1 > 2);
 print t = array_sort_desc( dynamic([null, null, null]) , false);
@@ -49,3 +90,62 @@ print array_sort_desc(dynamic(['1','3','4','5','2']),dynamic(["a","b","c","d","e
 print array_sort_desc(split("John,Paul,George,Ringo", ","));
 print array_sort_desc(dynamic([null,"blue","yellow","green",null]));
 print v=array_sort_desc(dynamic([null,"blue","yellow","green",null]), false);
+print '-- jaccard_index()';
+print jaccard_index(dynamic([1, 1, 2, 2, 3, 3]), dynamic([1, 2, 3, 4, 4, 4]));
+print jaccard_index(dynamic([1, 2, 3]), dynamic([]));
+print jaccard_index(dynamic([]), dynamic([1, 2, 3, 4]));
+print jaccard_index(dynamic([]), dynamic([]));
+print jaccard_index(dynamic([1, 2, 3]), dynamic([4, 5, 6, 7]));
+print jaccard_index(dynamic(['a', 's', 'd']), dynamic(['f', 'd', 's', 'a']));
+print jaccard_index(dynamic(['Chewbacca', 'Darth Vader', 'Han Solo']), dynamic(['Darth Sidious', 'Darth Vader']));
+print '-- pack_array()';
+print pack_array(); -- { serverError }
+print x = 1, y = x * 2, z = y * 2, pack_array(x,y,z);
+print pack_array(strcat('a', 'b'), format_ipv4(42), tostring(4.2));
+print '-- repeat()';
+print repeat(1, 0);
+print repeat(1, 3);
+print repeat("asd", 3);
+print repeat(timespan(1d), 3);
+print repeat(true, 3);
+print '-- set_difference()';
+print set_difference(dynamic([]), dynamic([]));
+print set_difference(dynamic([]), dynamic([9]));
+print set_difference(dynamic([]), dynamic(["asd"]));
+print set_difference(dynamic([1, 1, 2, 2, 3, 3]), dynamic([1, 2, 3]));
+print array_sort_asc(set_difference(dynamic([1, 4, 2, 3, 5, 4, 6]), dynamic([1, 2, 3])))[1];
+print set_difference(dynamic([4]), dynamic([1, 2, 3]));
+print array_sort_asc(set_difference(dynamic([1, 2, 3, 4, 5]), dynamic([5]), dynamic([2, 4])))[1];
+print array_sort_asc(set_difference(dynamic([1, 2, 3]), dynamic([])))[1];
+print array_sort_asc(set_difference(dynamic(['a', 's', 'd']), dynamic(['a', 'f'])))[1];
+print array_sort_asc(set_difference(dynamic(['Chewbacca', 'Darth Vader', 'Han Solo']), dynamic(['Darth Sidious', 'Darth Vader'])))[1];
+print '-- set_has_element()';
+print set_has_element(dynamic([]), 9);
+print set_has_element(dynamic(["this", "is", "an", "example"]), "example");
+print set_has_element(dynamic(["this", "is", "an", "example"]), "examplee");
+print set_has_element(dynamic([1, 2, 3]), 2);
+print set_has_element(dynamic([1, 2, 3, 4.2]), 4);
+print '-- set_intersect()';
+print set_intersect(dynamic([]), dynamic([]));
+print array_sort_asc(set_intersect(dynamic([1, 1, 2, 2, 3, 3]), dynamic([1, 2, 3])))[1];
+print array_sort_asc(set_intersect(dynamic([1, 4, 2, 3, 5, 4, 6]), dynamic([1, 2, 3])))[1];
+print set_intersect(dynamic([4]), dynamic([1, 2, 3]));
+print set_intersect(dynamic([1, 2, 3, 4, 5]), dynamic([1, 3, 5]), dynamic([2, 5]));
+print set_intersect(dynamic([1, 2, 3]), dynamic([]));
+print set_intersect(dynamic(['a', 's', 'd']), dynamic(['a', 'f']));
+print set_intersect(dynamic(['Chewbacca', 'Darth Vader', 'Han Solo']), dynamic(['Darth Sidious', 'Darth Vader']));
+print '-- set_union()';
+print set_union(dynamic([]), dynamic([]));
+print array_sort_asc(set_union(dynamic([1, 1, 2, 2, 3, 3]), dynamic([1, 2, 3])))[1];
+print array_sort_asc(set_union(dynamic([1, 4, 2, 3, 5, 4, 6]), dynamic([1, 2, 3])))[1];
+print array_sort_asc(set_union(dynamic([4]), dynamic([1, 2, 3])))[1];
+print array_sort_asc(set_union(dynamic([1, 3, 4]), dynamic([5]), dynamic([2, 4])))[1];
+print array_sort_asc(set_union(dynamic([1, 2, 3]), dynamic([])))[1];
+print array_sort_asc(set_union(dynamic(['a', 's', 'd']), dynamic(['a', 'f'])))[1];
+print array_sort_asc(set_union(dynamic(['Chewbacca', 'Darth Vader', 'Han Solo']), dynamic(['Darth Sidious', 'Darth Vader'])))[1];
+print '-- zip()';
+print zip(dynamic([]), dynamic([]));
+print zip(dynamic([1,3,5]), dynamic([2,4,6]));
+print zip(dynamic(['Darth','Master']), dynamic(['Vader','Yoda']), dynamic(['has a suit','doesn\'t have a suit']));
+print zip(dynamic([1,2,3]), dynamic([10,20]));
+print zip(dynamic([]), dynamic([1,2,3]));

tests/queries/0_stateless/02366_kql_math.reference

@@ -0,0 +1,4 @@
+-- isnan --
+1
+0
+0

tests/queries/0_stateless/02366_kql_math.sql

@@ -0,0 +1,7 @@
+set dialect = 'kusto';
+print '-- isnan --';
+print isnan(double(nan));
+print isnan(4.2);
+print isnan(4); -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }
+print isnan(real(+inf));
+print isnan(dynamic(null)); -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }