Implement functional tests for KQL functions

(cherry picked from commit 46ee780)

tests/queries/0_stateless/02366_kql_extend.reference

@@ -0,0 +1,17 @@
+-- extend #1 --
+Aldi	Apple	4	2016-09-10	400
+Costco	Apple	2	2016-09-11	200
+-- extend #2 --
+Apple	200
+Apple	400
+-- extend #3 --
+Apple cost 480 on average based on 5 samples.
+Snargaluff cost 28080 on average based on 5 samples.
+-- extend #4 --
+1
+-- extend #5 --
+Aldi	Apple	4	2016-09-10	Apple was purchased from Aldi for $4 on 2016-09-10	400
+Costco	Apple	2	2016-09-11	Apple was purchased from Costco for $2 on 2016-09-11	200
+-- extend #6 --
+-- extend #7 --
+-- extend #8 --

tests/queries/0_stateless/02366_kql_extend.sql

@@ -0,0 +1,49 @@
+-- datatable(Supplier:string, Fruit:string, Price: real, Purchase:datetime)
+-- [
+--     'Aldi','Apple',4,'2016-09-10',
+--     'Costco','Apple',2,'2016-09-11',
+--     'Aldi','Apple',6,'2016-09-10',
+--     'Costco','Snargaluff',100,'2016-09-12',
+--     'Aldi','Apple',7,'2016-09-12',
+--     'Aldi','Snargaluff',400,'2016-09-11',
+--     'Costco','Snargaluff',104,'2016-09-12',
+--     'Aldi','Apple',5,'2016-09-12',
+--     'Aldi','Snargaluff',600,'2016-09-11',
+--     'Costco','Snargaluff',200,'2016-09-10',
+-- ]
+
+DROP TABLE IF EXISTS Ledger;
+CREATE TABLE Ledger
+(    
+   Supplier Nullable(String),
+   Fruit String ,
+   Price Float64,
+   Purchase Date 
+) ENGINE = Memory;
+INSERT INTO Ledger VALUES  ('Aldi','Apple',4,'2016-09-10'), ('Costco','Apple',2,'2016-09-11'), ('Aldi','Apple',6,'2016-09-10'), ('Costco','Snargaluff',100,'2016-09-12'), ('Aldi','Apple',7,'2016-09-12'), ('Aldi','Snargaluff',400,'2016-09-11'),('Costco','Snargaluff',104,'2016-09-12'),('Aldi','Apple',5,'2016-09-12'),('Aldi','Snargaluff',600,'2016-09-11'),('Costco','Snargaluff',200,'2016-09-10');
+
+set dialect = 'kusto';
+
+print '-- extend #1 --';
+Ledger | extend PriceInCents = 100 * Price | take 2;
+
+print '-- extend #2 --';
+Ledger | extend PriceInCents = 100 * Price | sort by PriceInCents asc | project Fruit, PriceInCents | take 2;
+
+print '-- extend #3 --';
+Ledger | extend PriceInCents = 100 * Price | sort by PriceInCents asc | project Fruit, PriceInCents | summarize AveragePrice = avg(PriceInCents), Purchases = count() by Fruit | extend Sentence = strcat(Fruit, ' cost ', tostring(AveragePrice), ' on average based on ', tostring(Purchases), ' samples.') | project Sentence;
+
+print '-- extend #4 --';
+Ledger | extend a = Price | extend b = a | extend c = a, d = b + 500 | extend Pass = bool(b == a and c == a and d == b + 500) | summarize binary_all_and(Pass);
+
+print '-- extend #5 --';
+Ledger | take 2 | extend strcat(Fruit, ' was purchased from ', Supplier, ' for $', tostring(Price), ' on ', tostring(Purchase)) | extend PriceInCents = 100 * Price;
+
+print '-- extend #6 --';
+-- Ledger | extend Price = 100 * Price
+
+print '-- extend #7 --';
+-- print a = 4 | extend a = 5
+
+print '-- extend #8 --';
+-- print x = 5 | extend array_sort_desc(range(0, x), range(1, x + 1))

tests/queries/0_stateless/02366_kql_makeseries.sql

@@ -68,4 +68,7 @@ make_series_test_table2 | make-series PriceAvg=avg(Price) default=0 on Purchase
 print '-- without by';
 make_series_test_table2 | make-series PriceAvg=avg(Price) default=0 on Purchase step  2.0;
 
-make_series_test_table3 | make-series avg(metric) default=0  on timestamp from datetime(2017-01-01) to datetime(2017-01-10) step 1d 
+make_series_test_table3 | make-series avg(metric) default=0  on timestamp from datetime(2017-01-01) to datetime(2017-01-10) step 1d 
+
+-- print '-- summarize --'
+-- make_series_test_table | summarize count() by format_datetime(bin(Purchase, 1d), 'yy-MM-dd');

tests/queries/0_stateless/02366_kql_mvexpend.reference

@@ -1,12 +1,15 @@
--- mv-expend
+-- mv-expand --
+-- mv_expand_test_table | mv-expand c --
 1	['Salmon','Steak','Chicken']	1	[5,6,7,8]
 1	['Salmon','Steak','Chicken']	2	[5,6,7,8]
 1	['Salmon','Steak','Chicken']	3	[5,6,7,8]
 1	['Salmon','Steak','Chicken']	4	[5,6,7,8]
+-- mv_expand_test_table | mv-expand c, d --
 1	['Salmon','Steak','Chicken']	1	5
 1	['Salmon','Steak','Chicken']	2	6
 1	['Salmon','Steak','Chicken']	3	7
 1	['Salmon','Steak','Chicken']	4	8
+-- mv_expand_test_table | mv-expand b | mv-expand c --
 1	Salmon	1	[5,6,7,8]
 1	Salmon	2	[5,6,7,8]
 1	Salmon	3	[5,6,7,8]
@@ -19,10 +22,12 @@
 1	Chicken	2	[5,6,7,8]
 1	Chicken	3	[5,6,7,8]
 1	Chicken	4	[5,6,7,8]
+-- mv_expand_test_table | mv-expand with_itemindex=index b, c, d --
 0	1	Salmon	1	5
 1	1	Steak	2	6
 2	1	Chicken	3	7
 3	1		4	8
+-- mv_expand_test_table | mv-expand array_concat(c,d) --
 1	['Salmon','Steak','Chicken']	[1,2,3,4]	[5,6,7,8]	1
 1	['Salmon','Steak','Chicken']	[1,2,3,4]	[5,6,7,8]	2
 1	['Salmon','Steak','Chicken']	[1,2,3,4]	[5,6,7,8]	3
@@ -31,10 +36,12 @@
 1	['Salmon','Steak','Chicken']	[1,2,3,4]	[5,6,7,8]	6
 1	['Salmon','Steak','Chicken']	[1,2,3,4]	[5,6,7,8]	7
 1	['Salmon','Steak','Chicken']	[1,2,3,4]	[5,6,7,8]	8
+-- mv_expand_test_table | mv-expand x = c, y = d --
 1	['Salmon','Steak','Chicken']	[1,2,3,4]	[5,6,7,8]	1	5
 1	['Salmon','Steak','Chicken']	[1,2,3,4]	[5,6,7,8]	2	6
 1	['Salmon','Steak','Chicken']	[1,2,3,4]	[5,6,7,8]	3	7
 1	['Salmon','Steak','Chicken']	[1,2,3,4]	[5,6,7,8]	4	8
+-- mv_expand_test_table | mv-expand xy = array_concat(c, d) --
 1	['Salmon','Steak','Chicken']	[1,2,3,4]	[5,6,7,8]	1
 1	['Salmon','Steak','Chicken']	[1,2,3,4]	[5,6,7,8]	2
 1	['Salmon','Steak','Chicken']	[1,2,3,4]	[5,6,7,8]	3
@@ -43,9 +50,16 @@
 1	['Salmon','Steak','Chicken']	[1,2,3,4]	[5,6,7,8]	6
 1	['Salmon','Steak','Chicken']	[1,2,3,4]	[5,6,7,8]	7
 1	['Salmon','Steak','Chicken']	[1,2,3,4]	[5,6,7,8]	8
+-- mv_expand_test_table | mv-expand xy = array_concat(c, d) limit 2| summarize count() by xy --
 1	1
 2	1
+-- mv_expand_test_table | mv-expand with_itemindex=index c,d to typeof(bool) --
 0	1	['Salmon','Steak','Chicken']	1	true
 1	1	['Salmon','Steak','Chicken']	2	true
 2	1	['Salmon','Steak','Chicken']	3	true
 3	1	['Salmon','Steak','Chicken']	4	true
+-- mv_expand_test_table | mv-expand c to typeof(bool) --
+1	['Salmon','Steak','Chicken']	[5,6,7,8]	true
+1	['Salmon','Steak','Chicken']	[5,6,7,8]	true
+1	['Salmon','Steak','Chicken']	[5,6,7,8]	true
+1	['Salmon','Steak','Chicken']	[5,6,7,8]	true

tests/queries/0_stateless/02366_kql_mvexpend.sql

@@ -1,21 +1,35 @@
-DROP TABLE IF EXISTS mv_expend_test_table;
-CREATE TABLE mv_expend_test_table
+-- datatable(a: int, b: dynamic, c: dynamic, d: dynamic) [
+--    1, dynamic(['Salmon', 'Steak', 'Chicken']), dynamic([1, 2, 3, 4]), dynamic([5, 6, 7, 8])
+-- ]
+
+DROP TABLE IF EXISTS mv_expand_test_table;
+CREATE TABLE mv_expand_test_table
 (    
    a UInt8,
    b Array(String),
    c Array(Int8),
    d Array(Int8)
 ) ENGINE = Memory;
-INSERT INTO mv_expend_test_table VALUES (1, ['Salmon', 'Steak','Chicken'],[1,2,3,4],[5,6,7,8]);
+INSERT INTO mv_expand_test_table VALUES (1, ['Salmon', 'Steak','Chicken'],[1,2,3,4],[5,6,7,8]);
 set dialect='kusto';
-print '-- mv-expend';
-mv_expend_test_table | mv-expand c;
-mv_expend_test_table | mv-expand c, d;
-mv_expend_test_table | mv-expand b | mv-expand c;
-mv_expend_test_table | mv-expand with_itemindex=index b, c, d;
-mv_expend_test_table | mv-expand array_concat(c,d);
-mv_expend_test_table | mv-expand x = c, y = d;
-mv_expend_test_table | mv-expand xy = array_concat(c, d);
-mv_expend_test_table | mv-expand xy = array_concat(c, d) limit 2| summarize count() by xy;
-mv_expend_test_table | mv-expand with_itemindex=index c,d to typeof(bool);
--- mv_expend_test_table | mv-expand c to typeof(bool);
+print '-- mv-expand --';
+print '-- mv_expand_test_table | mv-expand c --';
+mv_expand_test_table | mv-expand c;
+print '-- mv_expand_test_table | mv-expand c, d --';
+mv_expand_test_table | mv-expand c, d;
+print '-- mv_expand_test_table | mv-expand b | mv-expand c --';
+mv_expand_test_table | mv-expand b | mv-expand c;
+print '-- mv_expand_test_table | mv-expand with_itemindex=index b, c, d --';
+mv_expand_test_table | mv-expand with_itemindex=index b, c, d;
+print '-- mv_expand_test_table | mv-expand array_concat(c,d) --';
+mv_expand_test_table | mv-expand array_concat(c,d);
+print '-- mv_expand_test_table | mv-expand x = c, y = d --';
+mv_expand_test_table | mv-expand x = c, y = d;
+print '-- mv_expand_test_table | mv-expand xy = array_concat(c, d) --';
+mv_expand_test_table | mv-expand xy = array_concat(c, d);
+print '-- mv_expand_test_table | mv-expand xy = array_concat(c, d) limit 2| summarize count() by xy --';
+mv_expand_test_table | mv-expand xy = array_concat(c, d) limit 2| summarize count() by xy;
+print '-- mv_expand_test_table | mv-expand with_itemindex=index c,d to typeof(bool) --';
+mv_expand_test_table | mv-expand with_itemindex=index c,d to typeof(bool);
+print '-- mv_expand_test_table | mv-expand c to typeof(bool) --';
+mv_expand_test_table | mv-expand c to typeof(bool);

tests/queries/0_stateless/02366_kql_summarize.reference

@@ -23,35 +23,60 @@ Management abcd defg	33
 40	2
 30	4
 20	6
+Skilled Manual	5
+Professional	6
+Management abcd defg	1
+-- make_list() --
 Skilled Manual	['Bachelors','Graduate Degree','High School','Partial College','Bachelors']
 Professional	['Graduate Degree','Partial College','Partial College','Partial College','Partial College','Partial College']
 Management abcd defg	['Bachelors']
 Skilled Manual	['Bachelors','Graduate Degree']
 Professional	['Graduate Degree','Partial College']
 Management abcd defg	['Bachelors']
+-- make_list_if() --
 Skilled Manual	['Edward','Christine']
 Professional	['Dalton','Angel']
 Management abcd defg	['Stephanie']
 Skilled Manual	['Edward']
 Professional	['Dalton']
 Management abcd defg	['Stephanie']
+-- make_set() --
 Skilled Manual	['Graduate Degree','High School','Partial College','Bachelors']
 Professional	['Graduate Degree','Partial College']
 Management abcd defg	['Bachelors']
 Skilled Manual	['Graduate Degree','Bachelors']
 Professional	['Graduate Degree','Partial College']
 Management abcd defg	['Bachelors']
+-- make_set_if() --
 Skilled Manual	['Partial College','High School']
 Professional	['Partial College']
 Management abcd defg	['Bachelors']
 Skilled Manual	['High School']
 Professional	['Partial College']
 Management abcd defg	['Bachelors']
+-- stdev() --
 6.855102059227432
+-- stdevif() --
 7.557189365836421
+-- binary_all_and --
 42
+-- binary_all_or --
 46
+-- binary_all_xor --
 4
 43.8
 [25.549999999999997,30.5,43.8]
 25.549999999999997
+35
+[25,35,45]
+-- Summarize following sort --
+Skilled Manual	5
+Professional	6
+Management abcd defg	1
+-- summarize with bin --
+0	1
+245000	2
+0	1
+245	2
+0	1
+245	2

tests/queries/0_stateless/02366_kql_summarize.sql

@@ -1,3 +1,18 @@
+-- datatable(FirstName:string, LastName:string, Occupation:string, Education:string, Age:int) [
+--     'Theodore', 'Diaz', 'Skilled Manual', 'Bachelors', 28, 
+--     'Stephanie', 'Cox', 'Management abcd defg', 'Bachelors', 33, 
+--     'Peter', 'Nara', 'Skilled Manual', 'Graduate Degree', 26, 
+--     'Latoya', 'Shen', 'Professional', 'Graduate Degree', 25, 
+--     'Joshua', 'Lee', 'Professional', 'Partial College', 26, 
+--     'Edward', 'Hernandez', 'Skilled Manual', 'High School', 36, 
+--     'Dalton', 'Wood', 'Professional', 'Partial College', 42, 
+--     'Christine', 'Nara', 'Skilled Manual', 'Partial College', 33, 
+--     'Cameron', 'Rodriguez', 'Professional', 'Partial College', 28, 
+--     'Angel', 'Stewart', 'Professional', 'Partial College', 46, 
+--     'Apple', '', 'Skilled Manual', 'Bachelors', 28, 
+--     dynamic(null), 'why', 'Professional', 'Partial College', 38
+-- ]
+
 DROP TABLE IF EXISTS Customers;
 CREATE TABLE Customers
 (    
@@ -8,21 +23,16 @@ CREATE TABLE Customers
     Age Nullable(UInt8)
 ) ENGINE = Memory;
 
--- INSERT INTO Customers VALUES  ('Theodore','Diaz','Skilled Manual','Bachelors',28);
--- INSERT INTO Customers VALUES  ('Stephanie','Cox','Management abcd defg','Bachelors',33);
--- INSERT INTO Customers VALUES  ('Peter','Nara','Skilled Manual','Graduate Degree',26);
--- INSERT INTO Customers VALUES  ('Latoya','Shen','Professional','Graduate Degree',25);
--- INSERT INTO Customers VALUES  ('Joshua','Lee','Professional','Partial College',26);
--- INSERT INTO Customers VALUES  ('Edward','Hernandez','Skilled Manual','High School',36);
--- INSERT INTO Customers VALUES  ('Dalton','Wood','Professional','Partial College',42);
--- INSERT INTO Customers VALUES  ('Christine','Nara','Skilled Manual','Partial College',33);
--- INSERT INTO Customers VALUES  ('Cameron','Rodriguez','Professional','Partial College',28);
--- INSERT INTO Customers VALUES  ('Angel','Stewart','Professional','Partial College',46);
--- INSERT INTO Customers VALUES  ('Apple','','Skilled Manual','Bachelors',28);
--- INSERT INTO Customers VALUES  (NULL,'why','Professional','Partial College',38);
-
 INSERT INTO Customers VALUES  ('Theodore','Diaz','Skilled Manual','Bachelors',28),('Stephanie','Cox','Management abcd defg','Bachelors',33),('Peter','Nara','Skilled Manual','Graduate Degree',26),('Latoya','Shen','Professional','Graduate Degree',25),('Joshua','Lee','Professional','Partial College',26),('Edward','Hernandez','Skilled Manual','High School',36),('Dalton','Wood','Professional','Partial College',42),('Christine','Nara','Skilled Manual','Partial College',33),('Cameron','Rodriguez','Professional','Partial College',28),('Angel','Stewart','Professional','Partial College',46),('Apple','','Skilled Manual','Bachelors',28),(NULL,'why','Professional','Partial College',38);
 
+drop table if exists EventLog;
+create table EventLog
+(
+    LogEntry String,
+    Created Int64
+) ENGINE = Memory;
+
+insert into EventLog values ('Darth Vader has entered the room.', 546), ('Rambo is suspciously looking at Darth Vader.', 245234), ('Darth Sidious electrocutes both using Force Lightning.', 245554);
 
 Select '-- test summarize --' ;
 set dialect='kusto';
@@ -36,38 +46,46 @@ Customers | summarize MySum = sumif(Age, Age<40) by Occupation;
 Customers | summarize dcount(Education);
 Customers | summarize dcountif(Education, Occupation=='Professional');
 Customers | summarize count_ = count() by bin(Age, 10) | order by count_ asc;
+Customers | summarize job_count = count() by Occupation | where job_count > 0;
 
--- make_list()
+print '-- make_list() --';
 Customers | summarize f_list = make_list(Education) by Occupation;
 Customers | summarize f_list = make_list(Education, 2) by Occupation;
--- make_list_if()
+print '-- make_list_if() --';
 Customers | summarize f_list = make_list_if(FirstName, Age>30) by Occupation;
 Customers | summarize f_list = make_list_if(FirstName, Age>30, 1) by Occupation;
--- make_set()
+print '-- make_set() --';
 Customers | summarize f_list = make_set(Education) by Occupation;
 Customers | summarize f_list = make_set(Education, 2) by Occupation;
--- make_set_if()
+print '-- make_set_if() --';
 Customers | summarize f_list = make_set_if(Education, Age>30) by Occupation;
 Customers | summarize f_list = make_set_if(Education, Age>30, 1) by Occupation;
--- stdev()
+print '-- stdev() --';
 Customers | project Age | summarize stdev(Age);
--- stdevif()
+print '-- stdevif() --';
 Customers | project Age | summarize stdevif(Age, Age%2==0);
--- binary_all_and
+print '-- binary_all_and --';
 Customers | project Age | where Age > 40 | summarize binary_all_and(Age);
--- binary_all_or
+print '-- binary_all_or --';
 Customers | project Age | where Age > 40 | summarize binary_all_or(Age);
--- binary_all_xor
+print '-- binary_all_xor --';
 Customers | project Age | where Age > 40 | summarize binary_all_xor(Age);
 
--- TODO:
 Customers | project Age | summarize percentile(Age, 95);
 Customers | project Age | summarize percentiles(Age, 5, 50, 95);
 Customers | project Age | summarize percentiles(Age, 5, 50, 95)[1];
--- Customers | summarize w=count() by AgeBucket=bin(Age, 5) | summarize percentilew(AgeBucket, w, 75); -- expect 35
--- Customers | summarize w=count() by AgeBucket=bin(Age, 5) | summarize percentilesw(AgeBucket, w, 50, 75, 99.9); -- expect 25,35,45
+Customers | summarize w=count() by AgeBucket=bin(Age, 5) | summarize percentilew(AgeBucket, w, 75);
+Customers | summarize w=count() by AgeBucket=bin(Age, 5) | summarize percentilesw(AgeBucket, w, 50, 75, 99.9);
+
+print '-- Summarize following sort --';
+Customers | sort by FirstName | summarize count() by Occupation;
 
+print '-- summarize with bin --';
+EventLog | summarize count=count() by bin(Created, 1000);
+EventLog | summarize count=count() by bin(unixtime_seconds_todatetime(Created/1000), 1s);
+EventLog | summarize count=count() by time_label=bin(Created/1000, 1s);
+
+-- TODO:
 -- arg_max()
 -- arg_min()
 -- make_list_with_nulls()
--- Customers | sort by FirstName | summarize count(Education) by Occupation;