Kusto-phase3: rebase v24.2.1.2248-stable

Kusto-phase3: rebase v24.2.1.2248-stable fixing tests.

Kusto-phase3: rebase v24.2.1.2248-stable fix has any ip functions.

Kusto-phase3: rebase v24.2.1.2248-stable provide additional protection on ip functions.

src/Common/IntervalKind.h

@@ -42,7 +42,7 @@ struct IntervalKind
     Float64 toSeconds() const;
 
     /// Chooses an interval kind based on number of seconds.
-    /// For example, `IntervalKind::fromAvgSeconds(3600)` returns `IntervalKind::Hour`.
+    /// For example, `IntervalKind::fromAvgSeconds(3600)` returns `IntervalKind::Kind::Hour`.
     static IntervalKind fromAvgSeconds(Int64 num_seconds);
 
     /// Returns whether IntervalKind has a fixed number of seconds (e.g. Day) or non-fixed(e.g. Month)
@@ -54,16 +54,16 @@ struct IntervalKind
     const char * toLowercasedKeyword() const;
 
     /// Returns the string which can be passed to the `unit` parameter of the dateDiff() function.
-    /// For example, `IntervalKind{IntervalKind::Day}.getDateDiffParameter()` returns "day".
+    /// For example, `IntervalKind{IntervalKind::Kind::Day}.getDateDiffParameter()` returns "day".
     const char * toDateDiffUnit() const;
 
     /// Returns the name of the function converting a number to the interval data type.
-    /// For example, `IntervalKind{IntervalKind::Day}.getToIntervalDataTypeFunctionName()`
+    /// For example, `IntervalKind{IntervalKind::Kind::Day}.getToIntervalDataTypeFunctionName()`
     /// returns "toIntervalDay".
     const char * toNameOfFunctionToIntervalDataType() const;
 
     /// Returns the name of the function extracting time part from a date or a time.
-    /// For example, `IntervalKind{IntervalKind::Day}.getExtractTimePartFunctionName()`
+    /// For example, `IntervalKind{IntervalKind::Kind::Day}.getExtractTimePartFunctionName()`
     /// returns "toDayOfMonth".
     const char * toNameOfFunctionExtractTimePart() const;
 

src/Core/SettingsChangesHistory.h

@@ -141,7 +141,9 @@ static std::map<ClickHouseVersion, SettingsChangesHistory::SettingsChanges> sett
     {"23.12", {{"allow_suspicious_ttl_expressions", true, false, "It is a new setting, and in previous versions the behavior was equivalent to allowing."},
               {"input_format_parquet_allow_missing_columns", false, true, "Allow missing columns in Parquet files by default"},
               {"input_format_orc_allow_missing_columns", false, true, "Allow missing columns in ORC files by default"},
-              {"input_format_arrow_allow_missing_columns", false, true, "Allow missing columns in Arrow files by default"}}},
+              {"input_format_arrow_allow_missing_columns", false, true, "Allow missing columns in Arrow files by default"},
+              {"split_parts_ranges_into_intersecting_and_non_intersecting_final", false, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"},
+              {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"}}},
     {"23.9", {{"optimize_group_by_constant_keys", false, true, "Optimize group by constant keys by default"},
               {"input_format_json_try_infer_named_tuples_from_objects", false, true, "Try to infer named Tuples from JSON objects by default"},
               {"input_format_json_read_numbers_as_strings", false, true, "Allow to read numbers as strings in JSON formats by default"},

src/Functions/FunctionBinaryArithmetic.h

@@ -1723,29 +1723,7 @@ class FunctionBinaryArithmetic : public IFunction
 
                 if constexpr (!std::is_same_v<ResultDataType, InvalidType>)
                 {
-                    if constexpr (is_div_int || is_div_int_or_zero)
-                        type_res = std::make_shared<ResultDataType>();
-                    else if constexpr (IsDataTypeDecimal<LeftDataType> && IsDataTypeDecimal<RightDataType>)
-                    {
-                        if constexpr (is_division)
-                        {
-                            if (context->getSettingsRef().decimal_check_overflow)
-                            {
-                                /// Check overflow by using operands scale (based on big decimal division implementation details):
-                                /// big decimal arithmetic is based on big integers, decimal operands are converted to big integers
-                                /// i.e. int_operand = decimal_operand*10^scale
-                                /// For division, left operand will be scaled by right operand scale also to do big integer division,
-                                /// BigInt result = left*10^(left_scale + right_scale) / right * 10^right_scale
-                                /// So, we can check upfront possible overflow just by checking max scale used for left operand
-                                /// Note: it doesn't detect all possible overflow during big decimal division
-                                if (left.getScale() + right.getScale() > ResultDataType::maxPrecision())
-                                    throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "Overflow during decimal division");
-                            }
-                        }
-                        ResultDataType result_type = decimalResultType<is_multiply, is_division>(left, right);
-                        type_res = std::make_shared<ResultDataType>(result_type.getPrecision(), result_type.getScale());
-                    }
-                    else if constexpr (((IsDataTypeDecimal<LeftDataType> && IsFloatingPoint<RightDataType>) ||
+                    if constexpr (((IsDataTypeDecimal<LeftDataType> && IsFloatingPoint<RightDataType>) ||
                         (IsDataTypeDecimal<RightDataType> && IsFloatingPoint<LeftDataType>)))
                     {
                         type_res = std::make_shared<DataTypeFloat64>();
@@ -1807,6 +1785,28 @@ class FunctionBinaryArithmetic : public IFunction
 
                         return static_cast<bool>(nested_type);
                     }
+                    else if constexpr (IsDataTypeDecimal<LeftDataType> && IsDataTypeDecimal<RightDataType>)
+                    {
+                        if constexpr (is_division)
+                        {
+                            if (context->getSettingsRef().decimal_check_overflow)
+                            {
+                                /// Check overflow by using operands scale (based on big decimal division implementation details):
+                                /// big decimal arithmetic is based on big integers, decimal operands are converted to big integers
+                                /// i.e. int_operand = decimal_operand*10^scale
+                                /// For division, left operand will be scaled by right operand scale also to do big integer division,
+                                /// BigInt result = left*10^(left_scale + right_scale) / right * 10^right_scale
+                                /// So, we can check upfront possible overflow just by checking max scale used for left operand
+                                /// Note: it doesn't detect all possible overflow during big decimal division
+                                if (left.getScale() + right.getScale() > ResultDataType::maxPrecision())
+                                    throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "Overflow during decimal division");
+                            }
+                        }
+                        ResultDataType result_type = decimalResultType<is_multiply, is_division>(left, right);
+                        type_res = std::make_shared<ResultDataType>(result_type.getPrecision(), result_type.getScale());
+                    }
+                    else if constexpr (is_div_int || is_div_int_or_zero)
+                        type_res = std::make_shared<ResultDataType>();
                     else
                         type_res = std::make_shared<ResultDataType>();
                     return true;

src/Functions/FunctionsConversion.h

@@ -3316,7 +3316,7 @@ class FunctionCast final : public FunctionCastBase
                         switch (interval_kind)
                         {
 #define DECLARE_CASE(NAME) \
-    case IntervalKind::NAME: \
+    case IntervalKind::Kind::NAME: \
         return FunctionToInterval##NAME::create();
                             FOR_EACH_INTERVAL_KIND(DECLARE_CASE)
 #undef DECLARE_CASE

src/Functions/Kusto/KqlRange.cpp

@@ -809,7 +809,7 @@ class FunctionKqlRange : public IFunction
                 total_values = total_elements;
         }
 
-        auto out = ColumnArray::create(std::make_shared<DataTypeInterval>(IntervalKind::Nanosecond)->createColumn());
+        auto out = ColumnArray::create(std::make_shared<DataTypeInterval>(IntervalKind::Kind::Nanosecond)->createColumn());
         IColumn & out_data = out->getData();
         IColumn::Offsets & out_offsets = out->getOffsets();
 

src/Functions/Kusto/kqlDateTime.cpp

@@ -95,11 +95,11 @@ ColumnPtr FunctionKqlDateTime<input_policy>::executeImpl(
 
     const ColumnsWithTypeAndName addition_args{
         asArgument(converted, "converted"),
-        createConstColumnWithTypeAndName<DataTypeInterval>(50, "interval_50", IntervalKind::Nanosecond)};
+        createConstColumnWithTypeAndName<DataTypeInterval>(50, "interval_50", IntervalKind::Kind::Nanosecond)};
     const auto sum = executeFunctionCall(context, "plus", addition_args, input_rows_count);
 
     const ColumnsWithTypeAndName to_start_of_interval_args{
-        asArgument(sum, "sum"), createConstColumnWithTypeAndName<DataTypeInterval>(100, "interval_100", IntervalKind::Nanosecond)};
+        asArgument(sum, "sum"), createConstColumnWithTypeAndName<DataTypeInterval>(100, "interval_100", IntervalKind::Kind::Nanosecond)};
     const auto [rounded_column, _] = executeFunctionCall(context, "toStartOfInterval", to_start_of_interval_args, input_rows_count);
 
     return wrapInNullable(rounded_column, conversion_args, result_type, input_rows_count);

src/Functions/Kusto/kqlHasAnyIp.cpp

@@ -1,9 +1,9 @@
 #include <optional>
 #include <ranges>
-#include <regex>
 #include <Columns/ColumnArray.h>
 #include <Columns/ColumnDecimal.h>
 #include <Columns/ColumnString.h>
+#include <Common/OptimizedRegularExpression.h>
 #include <DataTypes/DataTypeString.h>
 #include <DataTypes/DataTypesNumber.h>
 #include <Functions/FunctionFactory.h>
@@ -397,17 +397,26 @@ class FunctionKqlHasIpGeneric : public IFunction
             const auto ips = extractIpsFromArguments(arguments, result_type, context, i);
 
             std::string source = arguments[0].column->getDataAt(i).toString();
-            const std::regex ip_finder(Func::regex);
-            std::smatch matches;
 
-            while (!res && std::regex_search(source, matches, ip_finder))
+            OptimizedRegularExpression ip_finder(Func::regex);
+            OptimizedRegularExpression::MatchVec matches;
+
+            unsigned num_matches;
+            while (!res && (num_matches = ip_finder.match(source, matches)))
             {
-                res = Func::checkRegexMatch(matches[2].str(), result_type, context, ips);
+                if (num_matches <= 3) 
+                    continue;
 
-                source = matches.suffix().str();
+                res = Func::checkRegexMatch(source.substr(matches[2].offset, matches[2].length), result_type, context, ips);
+
+                if (matches[3].length == 0)
+                    source = "";
+                else
+                    source = source.substr(matches[3].offset, source.length());
             }
             result_column.push_back(static_cast<UInt8>(res));
         }
+
         return result;
     }
 

src/Functions/Kusto/kqlIndexOfRegex.cpp

@@ -1,21 +1,13 @@
 #include <Columns/ColumnNullable.h>
 #include <Columns/ColumnString.h>
 #include <Columns/ColumnsNumber.h>
+#include <Common/re2.h>
 #include <DataTypes/DataTypeString.h>
 #include <DataTypes/DataTypesNumber.h>
 #include <Functions/FunctionFactory.h>
 #include <Functions/FunctionHelpers.h>
 #include <Functions/IFunction.h>
 
-#ifdef __clang__
-#  pragma clang diagnostic push
-#  pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
-#endif
-#include <re2/re2.h>
-#ifdef __clang__
-#  pragma clang diagnostic pop
-#endif
-
 namespace DB::ErrorCodes
 {
 extern const int CANNOT_COMPILE_REGEXP;

src/Functions/Kusto/kqlToTimespan.cpp

@@ -79,7 +79,7 @@ ColumnPtr FunctionKqlToTimespan::executeImpl(
 
 DataTypePtr FunctionKqlToTimespan::getReturnTypeImpl(const DataTypes &) const
 {
-    return makeNullable(std::make_shared<DataTypeInterval>(IntervalKind::Nanosecond));
+    return makeNullable(std::make_shared<DataTypeInterval>(IntervalKind::Kind::Nanosecond));
 }
 
 REGISTER_FUNCTION(KqlToTimespan)

src/Functions/dateTime64Diff.cpp

@@ -92,7 +92,7 @@ DataTypePtr FunctionDateTime64Diff::getReturnTypeImpl(const DataTypes & argument
             rhs->getName(),
             getName());
 
-    return std::make_shared<DataTypeInterval>(IntervalKind::Nanosecond);
+    return std::make_shared<DataTypeInterval>(IntervalKind::Kind::Nanosecond);
 }
 
 REGISTER_FUNCTION(DateTime64Diff)

src/Interpreters/Context.cpp

@@ -15,6 +15,7 @@
 #include <Common/thread_local_rng.h>
 #include <Common/FieldVisitorToString.h>
 #include <Common/getMultipleKeysFromConfig.h>
+#include <Common/getNumberOfPhysicalCPUCores.h>
 #include <Common/callOnce.h>
 #include <Common/SharedLockGuard.h>
 #include <Coordination/KeeperDispatcher.h>

src/Parsers/ExpressionElementParsers.cpp

@@ -124,7 +124,7 @@ bool ParserSubquery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "EXPLAIN in a subquery cannot have a table function or table override");
 
         /// Replace subquery `(EXPLAIN <kind> <explain_settings> SELECT ...)`
-        /// with `(SELECT * FROM viewExplain("<kind>", "<explain_settings>", SELECT ...))`
+        /// with `(SELECT * FROM viewExplain('<kind>', '<explain_settings>', (SELECT ...)))`
 
         String kind_str = ASTExplainQuery::toString(explain_query.getKind());
 
@@ -142,7 +142,7 @@ bool ParserSubquery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
             auto view_explain = makeASTFunction("viewExplain",
                 std::make_shared<ASTLiteral>(kind_str),
                 std::make_shared<ASTLiteral>(settings_str),
-                explained_ast);
+                std::make_shared<ASTSubquery>(explained_ast));
             result_node = buildSelectFromTableFunction(view_explain);
         }
         else

src/Parsers/Kusto/KustoFunctions/IParserKQLFunction.cpp

@@ -364,7 +364,7 @@ String IParserKQLFunction::getExpression(IParser::Pos & pos)
                 arg = kqlTicksToInterval(ticks);
         }
     }
-    else if ((pos->type == TokenType::QuotedIdentifier))
+    else if (pos->type == TokenType::QuotedIdentifier)
         arg = "'" + escapeSingleQuotes(String(pos->begin + 1, pos->end - 1)) + "'";
     else if (pos->type == TokenType::At)
     {

src/Parsers/Kusto/KustoFunctions/KQLDataTypeFunctions.cpp

@@ -5,11 +5,10 @@
 
 #include <boost/lexical_cast/try_lexical_convert.hpp>
 #include <Poco/String.h>
+#include <Common/re2.h>
 
 #include <format>
-#include <regex>
 #include <unordered_set>
-#include <iostream>
 
 namespace DB::ErrorCodes
 {

src/Parsers/Kusto/KustoFunctions/KQLDateTimeFunctions.cpp

@@ -1,12 +1,12 @@
 #include "KQLDateTimeFunctions.h"
 
+#include <Parsers/Kusto/Utilities.h>
 #include <Common/StringUtils/StringUtils.h>
 
 #include <Poco/String.h>
 
 #include <format>
 #include <optional>
-#include <regex>
 #include <unordered_set>
 
 namespace

src/Parsers/Kusto/ParserKQLOperators.cpp

@@ -200,7 +200,7 @@ String genHasAnyAllOpExpr(
     auto haystack = tokens.back();
 
     const auto * const logic_op = (kql_op == "has_all") ? " and " : " or ";
-    while (isValidKQLPos(token_pos) && token_pos->type != TokenType::PipeMark && token_pos->type != TokenType::Semicolon)
+    while (isValidKQLPos(token_pos) && token_pos->type != DB::TokenType::PipeMark && token_pos->type != DB::TokenType::Semicolon)
     {
         auto tmp_arg = DB::IParserKQLFunction::getExpression(token_pos);
         if (token_pos->type == DB::TokenType::Comma)
@@ -217,14 +217,14 @@ String genHasAnyAllOpExpr(
     return new_expr;
 }
 
-String genEqOpExprCis(std::vector<String> & tokens, DB::IParser::Pos & token_pos, const DB::String & ch_op)
+String genEqOpExprCis(std::vector<String> & tokens, DB::IParser::Pos & token_pos, const String & ch_op)
 {
-    DB::String tmp_arg(token_pos->begin, token_pos->end);
+    String tmp_arg(token_pos->begin, token_pos->end);
 
     if (tokens.empty() || tmp_arg != "~")
         return tmp_arg;
 
-    DB::String new_expr;
+    String new_expr;
     new_expr += "lower(" + tokens.back() + ")" + " ";
     new_expr += ch_op + " ";
     ++token_pos;
@@ -240,7 +240,7 @@ String genEqOpExprCis(std::vector<String> & tokens, DB::IParser::Pos & token_pos
 
 String genBetweenOpExpr(std::vector<std::string> & tokens, DB::IParser::Pos & token_pos, const String & ch_op)
 {
-    DB::String new_expr;
+    String new_expr;
     new_expr += ch_op + "(";
     new_expr += tokens.back() + ",";
     tokens.pop_back();
@@ -289,15 +289,15 @@ String genBetweenOpExpr(std::vector<std::string> & tokens, DB::IParser::Pos & to
     return new_expr;
 }
 
-String genInOpExprCis(std::vector<String> & tokens, DB::IParser::Pos & token_pos, const DB::String & kql_op, const DB::String & ch_op)
+String genInOpExprCis(std::vector<String> & tokens, DB::IParser::Pos & token_pos, const String & kql_op, const String & ch_op)
 {
     DB::KQLContext kql_context;
     DB::ParserKQLTableFunction kqlfun_p(kql_context);
     DB::ParserToken s_lparen(DB::TokenType::OpeningRoundBracket);
 
     DB::ASTPtr select;
     DB::Expected expected;
-    DB::String new_expr;
+    String new_expr;
 
     ++token_pos;
     if (!s_lparen.ignore(token_pos, expected))
@@ -323,7 +323,7 @@ String genInOpExprCis(std::vector<String> & tokens, DB::IParser::Pos & token_pos
     new_expr += ch_op;
     while (isValidKQLPos(token_pos) && token_pos->type != DB::TokenType::PipeMark && token_pos->type != DB::TokenType::Semicolon)
     {
-        auto tmp_arg = DB::String(token_pos->begin, token_pos->end);
+        auto tmp_arg = String(token_pos->begin, token_pos->end);
         if (token_pos->type != DB::TokenType::Comma && token_pos->type != DB::TokenType::ClosingRoundBracket
             && token_pos->type != DB::TokenType::OpeningRoundBracket && token_pos->type != DB::TokenType::OpeningSquareBracket
             && token_pos->type != DB::TokenType::ClosingSquareBracket && tmp_arg != "~" && tmp_arg != "dynamic")

src/Parsers/Kusto/ParserKQLQuery.cpp

@@ -503,14 +503,18 @@ bool ParserKQLQuery::getOperations(Pos & pos, Expected & expected, OperationsPos
 
             if ((kql_operator == "print" || kql_operator == "range") && !operation_pos.empty())
                 throw Exception(ErrorCodes::SYNTAX_ERROR, "{} must be the first operator in the query", kql_operator);
+
+            operation_pos.emplace_back(kql_operator, pos);
         }
         else
             ++pos;
+    }
     return true;
 }
 
 bool ParserKQLQuery::pre_process(String & source, Pos & pos)
 {
+    bool need_preprocess = false;
     auto begin = pos;
     while (isValidKQLPos(pos) && pos->type != TokenType::Semicolon)
     {

src/Storages/System/StorageSystemSchema.cpp

@@ -24,15 +24,6 @@ namespace ErrorCodes
     extern const int LOGICAL_ERROR;
 }
 
-NamesAndTypesList StorageSystemSchema::getNamesAndTypes()
-{
-    return {
-        {"ColumnName", std::make_shared<DataTypeString>()},
-        {"ColumnOrdinal", std::make_shared<DataTypeInt32>()},
-        {"DataType", std::make_shared<DataTypeString>()},
-        {"ColumnType", std::make_shared<DataTypeString>()}};
-}
-
 void StorageSystemSchema::fillData(MutableColumns & res_columns, ContextPtr context, const SelectQueryInfo &) const
 {
     if (res_columns.size() != 4)
@@ -60,4 +51,15 @@ void StorageSystemSchema::fillData(MutableColumns & res_columns, ContextPtr cont
         column_types->insert(toField(is_kql ? toKQLDataTypeName(type) : type_name));
     }
 }
+
+ColumnsDescription StorageSystemSchema::getColumnsDescription()
+{
+    return ColumnsDescription
+    {
+        {"ColumnName", std::make_shared<DataTypeString>(), "Name of the column."},
+        {"ColumnOrdinal", std::make_shared<DataTypeInt32>(), "Ordinal of the column."},
+        {"DataType", std::make_shared<DataTypeString>(), "Data type of the column."},
+        {"ColumnType", std::make_shared<DataTypeString>(), "Type of the column."}
+    };
+}
 }