Skip to content

Commit

Permalink
Implement some KQL array functions
Browse files Browse the repository at this point in the history
  • Loading branch information
@ltrk2 @kashwy
ltrk2 authored and kashwy committed Aug 26, 2023
1 parent a803b48 commit c41f1cf
Show file tree
Hide file tree
Showing 10 changed files with 232 additions and 260 deletions.
13 changes: 12 additions & 1 deletion src/Parsers/Kusto/KQL_ReleaseNote.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,23 @@

# August 29, 2022
## Dynamic functions
- [array_concat](https://docs.microsoft.com/en-us/azure/data-explorer/kusto/query/arrayconcatfunction)
`print array_concat(dynamic([1, 2, 3]), dynamic([4, 5]), dynamic([6, 7, 8, 9])) == dynamic([1, 2, 3, 4, 5, 6, 7, 8, 9])`

- [array_iff / array_iif](https://docs.microsoft.com/en-us/azure/data-explorer/kusto/query/arrayifffunction)
`print array_iif(dynamic([true, false, true]), dynamic([1, 2, 3]), dynamic([4, 5, 6])) == dynamic([1, 5, 3])`
`print array_iif(dynamic([true, false, true]), dynamic([1, 2, 3, 4]), dynamic([4, 5, 6])) == dynamic([1, 5, 3])`
`print array_iif(dynamic([true, false, true, false]), dynamic([1, 2, 3, 4]), dynamic([4, 5, 6])) == dynamic([1, 5, 3, null])`
`print array_iif(dynamic([1, 0, -1, 44, 0]), dynamic([1, 2, 3, 4]), dynamic([4, 5, 6])) == dynamic([1, 5, 3, 4, null])`

- [array_slice](https://docs.microsoft.com/en-us/azure/data-explorer/kusto/query/arrayslicefunction)
`print array_slice(dynamic([1,2,3]), 1, 2) == dynamic([2, 3])`
`print array_slice(dynamic([1,2,3,4,5]), 2, -1) == dynamic([3, 4, 5])`
`print array_slice(dynamic([1,2,3,4,5]), -3, -2) == dynamic([3, 4])`

- [array_split](https://docs.microsoft.com/en-us/azure/data-explorer/kusto/query/arraysplitfunction)
`print array_split(dynamic([1,2,3,4,5]), 2) == dynamic([[1,2],[3,4,5]])`
`print array_split(dynamic([1,2,3,4,5]), dynamic([1,3])) == dynamic([[1],[2,3],[4,5]])`

## DateTimeFunctions

Expand Down Expand Up @@ -276,7 +287,7 @@
`print parse_ipv4_mask('192.1.168.2', 31) == 3221334018`
`print parse_ipv4_mask('192.1.168.3', 31) == 3221334018`
`print parse_ipv4_mask('127.2.3.4', 32) == 2130838276`
- [parse_ipv6_mask](https://docs.microsoft.com/en-us/azure/data-explorer/kusto/query/ipv6-comparefunction)
- [parse_ipv6_mask](https://docs.microsoft.com/en-us/azure/data-explorer/kusto/query/parse-ipv6-maskfunction)
`print parse_ipv6_mask('127.0.0.1', 24) == '0000:0000:0000:0000:0000:ffff:7f00:0000'`
`print parse_ipv6_mask('fe80::85d:e82c:9446:7994', 120) == 'fe80:0000:0000:0000:085d:e82c:9446:7900'`

Expand Down
167 changes: 92 additions & 75 deletions src/Parsers/Kusto/KustoFunctions/KQLDynamicFunctions.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,37 +1,29 @@
#include <Parsers/IParserBase.h>
#include <Parsers/ParserSetQuery.h>
#include <Parsers/ASTExpressionList.h>
#include <Parsers/ASTSelectWithUnionQuery.h>
#include <Parsers/Kusto/ParserKQLQuery.h>
#include <Parsers/Kusto/ParserKQLStatement.h>
#include <Parsers/IParserBase.h>
#include <Parsers/Kusto/KustoFunctions/IParserKQLFunction.h>
#include <Parsers/Kusto/KustoFunctions/KQLDateTimeFunctions.h>
#include <Parsers/Kusto/KustoFunctions/KQLStringFunctions.h>
#include <Parsers/Kusto/KustoFunctions/KQLDynamicFunctions.h>
#include <Parsers/Kusto/KustoFunctions/KQLCastingFunctions.h>
#include <Parsers/Kusto/KustoFunctions/KQLAggregationFunctions.h>
#include <Parsers/Kusto/KustoFunctions/KQLTimeSeriesFunctions.h>
#include <Parsers/Kusto/KustoFunctions/KQLIPFunctions.h>
#include <Parsers/Kusto/KustoFunctions/KQLBinaryFunctions.h>
#include <Parsers/Kusto/KustoFunctions/KQLCastingFunctions.h>
#include <Parsers/Kusto/KustoFunctions/KQLDateTimeFunctions.h>
#include <Parsers/Kusto/KustoFunctions/KQLDynamicFunctions.h>
#include <Parsers/Kusto/KustoFunctions/KQLGeneralFunctions.h>

#include <format>

namespace DB
{

bool ArrayConcat::convertImpl(String &out,IParser::Pos &pos)
bool ArrayConcat::convertImpl(String & out, IParser::Pos & pos)
{
String res = String(pos->begin,pos->end);
out = res;
return false;
return directMapping(out, pos, "arrayConcat");
}

bool ArrayIif::convertImpl(String & out, IParser::Pos & pos)
{
const auto function_name = getKQLFunctionName(pos);
if (function_name.empty())
return false;
return false;

const auto conditions = getArgument(function_name, pos);
const auto if_true = getArgument(function_name, pos);
Expand All @@ -46,189 +38,214 @@ bool ArrayIif::convertImpl(String & out, IParser::Pos & pos)
return true;
}

bool ArrayIndexOf::convertImpl(String &out,IParser::Pos &pos)
bool ArrayIndexOf::convertImpl(String & out, IParser::Pos & pos)
{
String fn_name = getKQLFunctionName(pos);

const auto fn_name = getKQLFunctionName(pos);
if (fn_name.empty())
return false;

++pos;
String array = getConvertedArgument(fn_name, pos);
++pos;
const auto needle = getConvertedArgument(fn_name, pos);
out = "minus(indexOf(" + array + ", " + needle + ") , 1)";

const auto array = getArgument(fn_name, pos);
const auto needle = getArgument(fn_name, pos);
out = "minus(indexOf(" + array + ", " + needle + "), 1)";

return true;
}

bool ArrayLength::convertImpl(String &out,IParser::Pos &pos)
bool ArrayLength::convertImpl(String & out, IParser::Pos & pos)
{
return directMapping(out, pos, "length");
}

bool ArrayReverse::convertImpl(String &out,IParser::Pos &pos)
bool ArrayReverse::convertImpl(String & out, IParser::Pos & pos)
{
String res = String(pos->begin,pos->end);
String res = String(pos->begin, pos->end);
out = res;
return false;
}

bool ArrayRotateLeft::convertImpl(String &out,IParser::Pos &pos)
bool ArrayRotateLeft::convertImpl(String & out, IParser::Pos & pos)
{
String res = String(pos->begin,pos->end);
String res = String(pos->begin, pos->end);
out = res;
return false;
}

bool ArrayRotateRight::convertImpl(String &out,IParser::Pos &pos)
bool ArrayRotateRight::convertImpl(String & out, IParser::Pos & pos)
{
String res = String(pos->begin,pos->end);
String res = String(pos->begin, pos->end);
out = res;
return false;
}

bool ArrayShiftLeft::convertImpl(String &out,IParser::Pos &pos)
bool ArrayShiftLeft::convertImpl(String & out, IParser::Pos & pos)
{
String res = String(pos->begin,pos->end);
String res = String(pos->begin, pos->end);
out = res;
return false;
}

bool ArrayShiftRight::convertImpl(String &out,IParser::Pos &pos)
bool ArrayShiftRight::convertImpl(String & out, IParser::Pos & pos)
{
String res = String(pos->begin,pos->end);
String res = String(pos->begin, pos->end);
out = res;
return false;
}

bool ArraySlice::convertImpl(String &out,IParser::Pos &pos)
bool ArraySlice::convertImpl(String & out, IParser::Pos & pos)
{
String res = String(pos->begin,pos->end);
out = res;
return false;
const auto function_name = getKQLFunctionName(pos);
if (function_name.empty())
return false;

const auto array = getArgument(function_name, pos);
const auto start = getArgument(function_name, pos);
const auto end = getArgument(function_name, pos);

out = std::format(
"arraySlice({0}, plus(1, if({1} >= 0, {1}, toInt64(max2(-length({0}), {1})) + length({0}))) as offset_{3}, "
" plus(1, if({2} >= 0, {2}, toInt64(max2(-length({0}), {2})) + length({0}))) - offset_{3} + 1)",
array,
start,
end,
generateUniqueIdentifier());

return true;
}

bool ArraySortAsc::convertImpl(String &out,IParser::Pos &pos)
bool ArraySortAsc::convertImpl(String & out, IParser::Pos & pos)
{
String res = String(pos->begin,pos->end);
String res = String(pos->begin, pos->end);
out = res;
return false;
}

bool ArraySortDesc::convertImpl(String &out,IParser::Pos &pos)
bool ArraySortDesc::convertImpl(String & out, IParser::Pos & pos)
{
String res = String(pos->begin,pos->end);
String res = String(pos->begin, pos->end);
out = res;
return false;
}

bool ArraySplit::convertImpl(String &out,IParser::Pos &pos)
bool ArraySplit::convertImpl(String & out, IParser::Pos & pos)
{
String res = String(pos->begin,pos->end);
out = res;
const auto function_name = getKQLFunctionName(pos);
if (function_name.empty())
return false;

const auto array = getArgument(function_name, pos);
const auto indices = getArgument(function_name, pos);

out = std::format(
"if(empty(arrayMap(x -> if(x >= 0, x, toInt64(max2(0, x + length({0})))), flatten([{1}])) as indices_{2}), [{0}], "
"arrayConcat([arraySlice({0}, 1, indices_{2}[1])], arrayMap(i -> arraySlice({0}, indices_{2}[i] + 1, "
"if(i = length(indices_{2}), length({0})::Int64, indices_{2}[i + 1]::Int64) - indices_{2}[i]), "
"range(1, length(indices_{2}) + 1))))",
array,
indices,
generateUniqueIdentifier());

return true;
}

bool ArraySum::convertImpl(String &out,IParser::Pos &pos)
bool ArraySum::convertImpl(String & out, IParser::Pos & pos)
{
return directMapping(out, pos, "arraySum");
}

bool BagKeys::convertImpl(String &out,IParser::Pos &pos)
bool BagKeys::convertImpl(String & out, IParser::Pos & pos)
{
String res = String(pos->begin,pos->end);
String res = String(pos->begin, pos->end);
out = res;
return false;
}

bool BagMerge::convertImpl(String &out,IParser::Pos &pos)
bool BagMerge::convertImpl(String & out, IParser::Pos & pos)
{
String res = String(pos->begin,pos->end);
String res = String(pos->begin, pos->end);
out = res;
return false;
}

bool BagRemoveKeys::convertImpl(String &out,IParser::Pos &pos)
bool BagRemoveKeys::convertImpl(String & out, IParser::Pos & pos)
{
String res = String(pos->begin,pos->end);
String res = String(pos->begin, pos->end);
out = res;
return false;
}

bool JaccardIndex::convertImpl(String &out,IParser::Pos &pos)
bool JaccardIndex::convertImpl(String & out, IParser::Pos & pos)
{
String res = String(pos->begin,pos->end);
String res = String(pos->begin, pos->end);
out = res;
return false;
}

bool Pack::convertImpl(String &out,IParser::Pos &pos)
bool Pack::convertImpl(String & out, IParser::Pos & pos)
{
String res = String(pos->begin,pos->end);
String res = String(pos->begin, pos->end);
out = res;
return false;
}

bool PackAll::convertImpl(String &out,IParser::Pos &pos)
bool PackAll::convertImpl(String & out, IParser::Pos & pos)
{
String res = String(pos->begin,pos->end);
String res = String(pos->begin, pos->end);
out = res;
return false;
}

bool PackArray::convertImpl(String &out,IParser::Pos &pos)
bool PackArray::convertImpl(String & out, IParser::Pos & pos)
{
String res = String(pos->begin,pos->end);
String res = String(pos->begin, pos->end);
out = res;
return false;
}

bool Repeat::convertImpl(String &out,IParser::Pos &pos)
bool Repeat::convertImpl(String & out, IParser::Pos & pos)
{
String res = String(pos->begin,pos->end);
String res = String(pos->begin, pos->end);
out = res;
return false;
}

bool SetDifference::convertImpl(String &out,IParser::Pos &pos)
bool SetDifference::convertImpl(String & out, IParser::Pos & pos)
{
String res = String(pos->begin,pos->end);
String res = String(pos->begin, pos->end);
out = res;
return false;
}

bool SetHasElement::convertImpl(String &out,IParser::Pos &pos)
bool SetHasElement::convertImpl(String & out, IParser::Pos & pos)
{
String res = String(pos->begin,pos->end);
String res = String(pos->begin, pos->end);
out = res;
return false;
}

bool SetIntersect::convertImpl(String &out,IParser::Pos &pos)
bool SetIntersect::convertImpl(String & out, IParser::Pos & pos)
{
String res = String(pos->begin,pos->end);
String res = String(pos->begin, pos->end);
out = res;
return false;
}

bool SetUnion::convertImpl(String &out,IParser::Pos &pos)
bool SetUnion::convertImpl(String & out, IParser::Pos & pos)
{
String res = String(pos->begin,pos->end);
String res = String(pos->begin, pos->end);
out = res;
return false;
}

bool TreePath::convertImpl(String &out,IParser::Pos &pos)
bool TreePath::convertImpl(String & out, IParser::Pos & pos)
{
String res = String(pos->begin,pos->end);
String res = String(pos->begin, pos->end);
out = res;
return false;
}

bool Zip::convertImpl(String &out,IParser::Pos &pos)
bool Zip::convertImpl(String & out, IParser::Pos & pos)
{
String res = String(pos->begin,pos->end);
String res = String(pos->begin, pos->end);
out = res;
return false;
}
Expand Down
6 changes: 4 additions & 2 deletions src/Parsers/Kusto/KustoFunctions/KQLIPFunctions.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,11 +225,13 @@ bool ParseIpv6Mask::convertImpl(String & out, IParser::Pos & pos)

const auto ip_address = getArgument(function_name, pos);
const auto mask = getArgument(function_name, pos);
const auto unique_identifier = generateUniqueIdentifier();
out = std::format(
"if(isNull({0} as ipv4), {1}, {2})",
"if(isNull({0} as ipv4_{3}), {1}, {2})",
kqlCallToExpression("parse_ipv4_mask", {ip_address, mask}, pos.max_depth),
kqlCallToExpression("parse_ipv6", {"strcat(tostring(parse_ipv6(" + ip_address + ")), '/', tostring(" + mask + "))"}, pos.max_depth),
kqlCallToExpression("parse_ipv6", {"format_ipv4(ipv4)"}, pos.max_depth));
kqlCallToExpression("parse_ipv6", {"format_ipv4(ipv4_" + unique_identifier + ")"}, pos.max_depth),
unique_identifier);
return true;
}

Expand Down
Loading

0 comments on commit c41f1cf

Please sign in to comment.