From fabcb94d102ed1bfd072bdc1acf49f8655022ef7 Mon Sep 17 00:00:00 2001 From: Grieve Date: Tue, 26 Nov 2024 16:16:55 +0800 Subject: [PATCH] test(clickhouse): test all functions --- .../resources/function_list/clickhouse.csv | 140 ++++-------------- .../v3/connector/clickhouse/test_functions.py | 20 ++- ibis-server/tests/util/sql_generator.py | 27 ++++ 3 files changed, 76 insertions(+), 111 deletions(-) diff --git a/ibis-server/resources/function_list/clickhouse.csv b/ibis-server/resources/function_list/clickhouse.csv index b54a4917a..efd7b5f55 100644 --- a/ibis-server/resources/function_list/clickhouse.csv +++ b/ibis-server/resources/function_list/clickhouse.csv @@ -1,110 +1,30 @@ -function_type,name,return_type,description -aggregate,count,UInt64,"Counts the number of rows or non-NULL values." -aggregate,countIf,UInt64,"Counts the number of rows that match a condition." -aggregate,min,Same as input,"Returns the minimum value." -aggregate,max,Same as input,"Returns the maximum value." -aggregate,sum,Numeric,"Returns the sum of values." -aggregate,avg,Float64,"Returns the average of values." -aggregate,uniq,UInt64,"Approximate number of different values using HyperLogLog." -aggregate,uniqExact,UInt64,"Exact number of different values." -aggregate,groupArray,Array,"Creates an array of values." -aggregate,quantile,Float64,"Calculates quantile using reservoir sampling." -aggregate,quantileDeterministic,Float64,"Deterministic calculation of quantile." -aggregate,quantileExact,Float64,"Exact calculation of quantile." -aggregate,quantileExactWeighted,Float64,"Exact calculation of weighted quantile." -aggregate,varPop,Float64,"Calculate population variance." -aggregate,varSamp,Float64,"Calculate sample variance." -aggregate,stddevPop,Float64,"Calculate population standard deviation." -aggregate,stddevSamp,Float64,"Calculate sample standard deviation." -aggregate,covarPop,Float64,"Calculate population covariance." -aggregate,covarSamp,Float64,"Calculate sample covariance." -aggregate,corrStable,Float64,"Calculate correlation coefficient." -aggregate,arrayUniq,UInt64,"Returns number of unique elements across arrays." -aggregate,sumArray,Array,"Sum of arrays element-wise." -aggregate,groupArrayArray,Array,"Combines multiple arrays into array of arrays." -scalar,toInt8,Int8,"Converts input to Int8 data type." -scalar,toInt16,Int16,"Converts input to Int16 data type." -scalar,toInt32,Int32,"Converts input to Int32 data type." -scalar,toInt64,Int64,"Converts input to Int64 data type." -scalar,toUInt8,UInt8,"Converts input to UInt8 data type." -scalar,toUInt16,UInt16,"Converts input to UInt16 data type." -scalar,toUInt32,UInt32,"Converts input to UInt32 data type." -scalar,toUInt64,UInt64,"Converts input to UInt64 data type." -scalar,toFloat32,Float32,"Converts input to Float32 data type." -scalar,toFloat64,Float64,"Converts input to Float64 data type." -scalar,toDate,Date,"Converts input to Date type." -scalar,toDateTime,DateTime,"Converts input to DateTime type." -scalar,toString,String,"Converts input to String type." -scalar,toDecimal32,Decimal32,"Converts input to Decimal32 type." -scalar,toDecimal64,Decimal64,"Converts input to Decimal64 type." -scalar,toDecimal128,Decimal128,"Converts input to Decimal128 type." -scalar,abs,Numeric,"Returns absolute value." -scalar,round,Same as input,"Rounds number to specified decimal places." -scalar,floor,Same as input,"Rounds down to nearest integer." -scalar,ceil,Same as input,"Rounds up to nearest integer." -scalar,exp,Float64,"Returns e raised to the power of x." -scalar,log,Float64,"Returns natural logarithm." -scalar,log2,Float64,"Returns base-2 logarithm." -scalar,log10,Float64,"Returns base-10 logarithm." -scalar,power,Float64,"Raises a number to the specified power." -scalar,sqrt,Float64,"Returns square root." -scalar,cbrt,Float64,"Returns cube root." -scalar,rand,UInt32,"Returns random number." -scalar,rand64,UInt64,"Returns random 64-bit number." -scalar,pi,Float64,"Returns value of π." -scalar,e,Float64,"Returns value of e." -scalar,match,UInt8,"Returns whether string matches regex pattern." -scalar,like,UInt8,"Pattern matching using LIKE operator." -scalar,notLike,UInt8,"Inverse of LIKE operator." -scalar,concat,String,"Concatenates strings." -scalar,substring,String,"Extracts substring." -scalar,substringUTF8,String,"Extracts substring considering UTF-8 encoding." -scalar,length,UInt64,"Returns string length in bytes." -scalar,lengthUTF8,UInt64,"Returns string length in Unicode characters." -scalar,lower,String,"Converts string to lowercase." -scalar,upper,String,"Converts string to uppercase." -scalar,trim,String,"Removes leading and trailing whitespace." -scalar,ltrim,String,"Removes leading whitespace." -scalar,rtrim,String,"Removes trailing whitespace." -scalar,reverse,String,"Reverses string." -scalar,replaceAll,String,"Replaces all occurrences of substring." -scalar,replaceRegexpAll,String,"Replaces all regex matches." -scalar,empty,UInt8,"Checks if array is empty." -scalar,notEmpty,UInt8,"Checks if array is not empty." -scalar,arrayLength,UInt64,"Returns length of array." -scalar,arrayConcat,Array,"Concatenates arrays." -scalar,arrayElement,Any,"Returns array element at specified index." -scalar,has,UInt8,"Checks if array contains value." -scalar,indexOf,UInt64,"Returns index of first occurrence of element." -scalar,toYear,UInt16,"Extracts year from date/datetime." -scalar,toQuarter,UInt8,"Extracts quarter from date/datetime." -scalar,toMonth,UInt8,"Extracts month from date/datetime." -scalar,toDayOfMonth,UInt8,"Extracts day of month from date/datetime." -scalar,toDayOfWeek,UInt8,"Extracts day of week from date/datetime." -scalar,toHour,UInt8,"Extracts hour from datetime." -scalar,toMinute,UInt8,"Extracts minute from datetime." -scalar,toSecond,UInt8,"Extracts second from datetime." -scalar,now,DateTime,"Returns current date and time." -scalar,today,Date,"Returns current date." -scalar,yesterday,Date,"Returns yesterday's date." -scalar,formatDateTime,String,"Formats date/time according to given format." -scalar,JSONHas,UInt8,"Checks if JSON field exists." -scalar,JSONLength,UInt64,"Returns length of JSON array or object." -scalar,JSONType,String,"Returns type of JSON value." -scalar,JSONExtractString,String,"Extracts string from JSON." -scalar,JSONExtractFloat,Float64,"Extracts float from JSON." -scalar,JSONExtractInt,Int64,"Extracts integer from JSON." -scalar,IPv4NumToString,String,"Converts IPv4 to string." -scalar,IPv4StringToNum,UInt32,"Converts string to IPv4." -scalar,IPv6NumToString,String,"Converts IPv6 to string." -scalar,IPv6StringToNum,FixedString(16),"Converts string to IPv6." -window,rowNumberInAllBlocks,UInt64,"Returns row number across all blocks." -window,rowNumberInBlock,UInt64,"Returns row number within current block." -window,rank,UInt64,"Returns rank of current row with gaps." -window,denseRank,UInt64,"Returns rank of current row without gaps." -window,runningDifference,Same as input,"Returns difference between successive values." -window,lagInFrame,Same as input,"Returns value from previous row in frame." -window,leadInFrame,Same as input,"Returns value from next row in frame." -window,firstValue,Same as input,"Returns first value in window frame." -window,lastValue,Same as input,"Returns last value in window frame." -window,nthValue,Same as input,"Returns nth value in window frame." +function_type,name,return_type,param_names,param_types,description +aggregate,min,Same as input,,Any,"Returns the minimum value." +aggregate,max,Same as input,,Any,"Returns the maximum value." +aggregate,sum,Numeric,,Numeric,"Returns the sum of values." +aggregate,avg,Float64,,Numeric,"Returns the average of values." +aggregate,uniq,UInt64,,Any,"Approximate number of different values using HyperLogLog." +scalar,abs,Numeric,,Numeric,"Returns absolute value." +scalar,round,Same as input,,Float64, "Rounds number to specified decimal places." +scalar,floor,Same as input,,Numeric,"Rounds down to nearest integer." +scalar,ceil,Same as input,,Numeric,"Rounds up to nearest integer." +scalar,exp,Float64,,Float64,"Returns e raised to the power of x." +scalar,log,Float64,,Float64,"Returns natural logarithm." +scalar,log2,Float64,,Float64,"Returns base-2 logarithm." +scalar,log10,Float64,,Float64,"Returns base-10 logarithm." +scalar,sqrt,Float64,,Float64,"Returns square root." +scalar,rand,UInt32,,,"Returns random number." +scalar,rand64,UInt64,,,"Returns random 64-bit number." +scalar,pi,Float64,,,"Returns value of π." +scalar,e,Float64,,,"Returns value of e." +scalar,concat,String,,Array, "Concatenates strings." +scalar,length,UInt64,,String,"Returns string length in bytes." +scalar,lower,String,,String,"Converts string to lowercase." +scalar,upper,String,,String,"Converts string to uppercase." +scalar,ltrim,String,,String,"Removes leading whitespace." +scalar,rtrim,String,,String,"Removes trailing whitespace." +scalar,reverse,String,,String,"Reverses string." +scalar,empty,UInt8,,Array, "Checks if array is empty." +scalar,now,DateTime,,,"Returns current date and time." +scalar,today,Date,,,"Returns current date." +scalar,yesterday,Date,,,"Returns yesterday's date." diff --git a/ibis-server/tests/routers/v3/connector/clickhouse/test_functions.py b/ibis-server/tests/routers/v3/connector/clickhouse/test_functions.py index 95ecdba41..d5f9a67b6 100644 --- a/ibis-server/tests/routers/v3/connector/clickhouse/test_functions.py +++ b/ibis-server/tests/routers/v3/connector/clickhouse/test_functions.py @@ -1,4 +1,5 @@ import base64 +import os import orjson import pytest @@ -8,6 +9,7 @@ from app.main import app from tests.conftest import DATAFUSION_FUNCTION_COUNT, file_path from tests.routers.v3.connector.clickhouse.conftest import base_url +from tests.util import FunctionCsvParser, SqlTestGenerator manifest = { "catalog": "my_catalog", @@ -57,7 +59,7 @@ def test_function_list(): response = client.get(url=f"{base_url}/functions") assert response.status_code == 200 result = response.json() - assert len(result) == DATAFUSION_FUNCTION_COUNT + 79 + assert len(result) == DATAFUSION_FUNCTION_COUNT + 5 the_func = next(filter(lambda x: x["name"] == "abs", result)) assert the_func == { "name": "abs", @@ -107,3 +109,19 @@ def test_aggregate_function(manifest_str: str, connection_info): "data": [[1]], "dtypes": {"col": "uint64"}, } + + def test_functions(manifest_str: str, connection_info): + csv_path = os.path.join(function_list_path, "clickhouse.csv") + csv_parser = FunctionCsvParser(csv_path) + sql_generator = SqlTestGenerator("clickhouse") + for function in csv_parser.parse(): + sql = sql_generator.generate_sql(function) + response = client.post( + url=f"{base_url}/query", + json={ + "connectionInfo": connection_info, + "manifestStr": manifest_str, + "sql": sql, + }, + ) + assert response.status_code == 200 diff --git a/ibis-server/tests/util/sql_generator.py b/ibis-server/tests/util/sql_generator.py index f9c9ba332..c652d2e66 100644 --- a/ibis-server/tests/util/sql_generator.py +++ b/ibis-server/tests/util/sql_generator.py @@ -26,6 +26,8 @@ def _get_generator(self): return BigQuerySqlGenerator() if self.dialect == "postgres": return PostgresSqlGenerator() + if self.dialect == "clickhouse": + return ClickhouseSqlGenerator() raise NotImplementedError(f"Unsupported dialect: {self.dialect}") @staticmethod @@ -150,3 +152,28 @@ def generate_window_sql(self, function: Function) -> str: SELECT 5 AS id, 'A' AS category ) AS t """ + + +class ClickhouseSqlGenerator(SqlGenerator): + def generate_aggregate_sql(self, function: Function) -> str: + sample_args = self.generate_sample_args(function.param_types) + formatted_args = ", ".join(sample_args) + return f"SELECT {function.name}({formatted_args})" + + def generate_scalar_sql(self, function: Function) -> str: + args = self.generate_sample_args(function.param_types) + formatted_args = ", ".join(args) + return f"SELECT {function.name}({formatted_args})" + + def generate_window_sql(self, function: Function) -> str: + return f""" + SELECT + {function.name}() OVER (ORDER BY id) AS {function.name.lower()} + FROM ( + SELECT 1 AS id, 'A' AS category UNION ALL + SELECT 2 AS id, 'B' AS category UNION ALL + SELECT 3 AS id, 'A' AS category UNION ALL + SELECT 4 AS id, 'B' AS category UNION ALL + SELECT 5 AS id, 'A' AS category + ) AS t + """