From 22162cefe23c8360c4ac9559dccc7863ee36272b Mon Sep 17 00:00:00 2001 From: Jax Liu Date: Fri, 3 Jan 2025 18:25:52 +0800 Subject: [PATCH 1/7] format float with scientific notation --- ibis-server/app/util.py | 4 +- .../routers/v2/connector/test_postgres.py | 81 +++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) diff --git a/ibis-server/app/util.py b/ibis-server/app/util.py index b80616857..071f7440b 100644 --- a/ibis-server/app/util.py +++ b/ibis-server/app/util.py @@ -29,7 +29,9 @@ def _to_datetime_and_format(series: pd.Series) -> pd.Series: def _to_json_obj(df: pd.DataFrame) -> dict: - data = df.to_dict(orient="split", index=False) + data = df.map(lambda x: f"{x:.9f}" if isinstance(x, float) else x).to_dict( + orient="split", index=False + ) def default(obj): if pd.isna(obj): diff --git a/ibis-server/tests/routers/v2/connector/test_postgres.py b/ibis-server/tests/routers/v2/connector/test_postgres.py index e64081fad..71e3b6ec5 100644 --- a/ibis-server/tests/routers/v2/connector/test_postgres.py +++ b/ibis-server/tests/routers/v2/connector/test_postgres.py @@ -208,6 +208,87 @@ async def test_query_with_dot_all(client, manifest_str, postgres: PostgresContai assert result["dtypes"] is not None +async def test_format_floating(client, manifest_str, postgres): + connection_info = _to_connection_info(postgres) + response = await client.post( + url=f"{base_url}/query", + json={ + "connectionInfo": connection_info, + "manifestStr": manifest_str, + "sql": """ +SELECT + 0.0123e-5 AS case_scientific_original, + 1.23e+4 AS case_scientific_positive, + -4.56e-3 AS case_scientific_negative, + 7.89e0 AS case_scientific_zero_exponent, + 0e0 AS case_scientific_zero, + + 123.456 AS case_decimal_positive, + -123.456 AS case_decimal_negative, + 0.0000123 AS case_decimal_small, + 123.0000 AS case_decimal_trailing_zeros, + 0.0 AS case_decimal_zero, + + 0 AS case_integer_zero, + 0e-9 AS case_integer_zero_scientific, + -1 AS case_integer_negative, + 9999999999 AS case_integer_large, + + 1.7976931348623157E+308 AS case_float_max, + 2.2250738585072014E-308 AS case_float_min, + -1.7976931348623157E+308 AS case_float_min_negative, + + 1.23e4 + 4.56 AS case_mixed_addition, + -1.23e-4 - 123.45 AS case_mixed_subtraction, + 0.0123e-5 * 1000 AS case_mixed_multiplication, + 123.45 / 1.23e2 AS case_mixed_division, + + CAST('NaN' AS FLOAT) AS case_special_nan, + CAST('Infinity' AS FLOAT) AS case_special_infinity, + CAST('-Infinity' AS FLOAT) AS case_special_negative_infinity, + NULL AS case_special_null, + + CAST(123.456 AS FLOAT) AS case_cast_float, + CAST(1.23e4 AS DECIMAL(10,5)) AS case_cast_decimal + """, + }, + ) + assert response.status_code == 200 + result = response.json() + + assert result["data"] == [ + [ + "1.23E-7", + "1.23E+4", + "-0.00456", + "7.89", + "0", + "123.456", + "-123.456", + "0.0000123", + "123", + "0", + 0, + "0", + -1, + 9999999999, + "1.7976931348623157E+308", + "2.2250738585072014E-308", + "-1.7976931348623157E+308", + "12304.56", + "-123.450123", + "0.000123", + "1.0036585365853659", + "nan", + "inf", + "-inf", + None, + "123.456001282", + "12300.00000", + ] + ] + + async def test_dry_run_with_connection_url_and_password_with_bracket_should_not_raise_value_error( client, manifest_str, postgres: PostgresContainer ): From b33d203fadfdc7c6576956575b693d8c4f6a1ceb Mon Sep 17 00:00:00 2001 From: Jax Liu Date: Fri, 3 Jan 2025 18:39:31 +0800 Subject: [PATCH 2/7] change fmt --- ibis-server/app/util.py | 2 +- ibis-server/tests/routers/v2/connector/test_postgres.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ibis-server/app/util.py b/ibis-server/app/util.py index 071f7440b..f05f61e2e 100644 --- a/ibis-server/app/util.py +++ b/ibis-server/app/util.py @@ -29,7 +29,7 @@ def _to_datetime_and_format(series: pd.Series) -> pd.Series: def _to_json_obj(df: pd.DataFrame) -> dict: - data = df.map(lambda x: f"{x:.9f}" if isinstance(x, float) else x).to_dict( + data = df.map(lambda x: f"{x:.9g}" if isinstance(x, float) else x).to_dict( orient="split", index=False ) diff --git a/ibis-server/tests/routers/v2/connector/test_postgres.py b/ibis-server/tests/routers/v2/connector/test_postgres.py index 71e3b6ec5..9c6407bf0 100644 --- a/ibis-server/tests/routers/v2/connector/test_postgres.py +++ b/ibis-server/tests/routers/v2/connector/test_postgres.py @@ -283,8 +283,8 @@ async def test_format_floating(client, manifest_str, postgres): "inf", "-inf", None, - "123.456001282", - "12300.00000", + "123.456001", + "12300.00000" ] ] From 2f3f04421375d3abbc2dddd0a99dbc6b1af9711f Mon Sep 17 00:00:00 2001 From: Jax Liu Date: Fri, 3 Jan 2025 18:41:30 +0800 Subject: [PATCH 3/7] fix fmt --- ibis-server/tests/routers/v2/connector/test_postgres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibis-server/tests/routers/v2/connector/test_postgres.py b/ibis-server/tests/routers/v2/connector/test_postgres.py index 9c6407bf0..5a441c17d 100644 --- a/ibis-server/tests/routers/v2/connector/test_postgres.py +++ b/ibis-server/tests/routers/v2/connector/test_postgres.py @@ -284,7 +284,7 @@ async def test_format_floating(client, manifest_str, postgres): "-inf", None, "123.456001", - "12300.00000" + "12300.00000", ] ] From c671235169f91685e7323f03b0d07d9f2dd02010 Mon Sep 17 00:00:00 2001 From: Jax Liu Date: Fri, 3 Jan 2025 19:13:19 +0800 Subject: [PATCH 4/7] refactor test --- .../routers/v2/connector/test_postgres.py | 58 +++++++++---------- 1 file changed, 27 insertions(+), 31 deletions(-) diff --git a/ibis-server/tests/routers/v2/connector/test_postgres.py b/ibis-server/tests/routers/v2/connector/test_postgres.py index 5a441c17d..09c36715b 100644 --- a/ibis-server/tests/routers/v2/connector/test_postgres.py +++ b/ibis-server/tests/routers/v2/connector/test_postgres.py @@ -256,37 +256,33 @@ async def test_format_floating(client, manifest_str, postgres): assert response.status_code == 200 result = response.json() - assert result["data"] == [ - [ - "1.23E-7", - "1.23E+4", - "-0.00456", - "7.89", - "0", - "123.456", - "-123.456", - "0.0000123", - "123", - "0", - 0, - "0", - -1, - 9999999999, - "1.7976931348623157E+308", - "2.2250738585072014E-308", - "-1.7976931348623157E+308", - "12304.56", - "-123.450123", - "0.000123", - "1.0036585365853659", - "nan", - "inf", - "-inf", - None, - "123.456001", - "12300.00000", - ] - ] + assert result["data"][0] == "1.23E-7" + assert result["data"][1] == "1.23E+4" + assert result["data"][2] == "-0.00456" + assert result["data"][3] == "7.89" + assert result["data"][4] == "0" + assert result["data"][5] == "123.456" + assert result["data"][6] == "-123.456" + assert result["data"][7] == "0.0000123" + assert result["data"][8] == "123" + assert result["data"][9] == "0" + assert result["data"][10] == 0 + assert result["data"][11] == "0" + assert result["data"][12] == -1 + assert result["data"][13] == 9999999999 + assert result["data"][14] == "1.7976931348623157E+308" + assert result["data"][15] == "2.2250738585072014E-308" + assert result["data"][16] == "-1.7976931348623157E+308" + assert result["data"][17] == "12304.56" + assert result["data"][18] == "-123.450123" + assert result["data"][19] == "0.000123" + assert result["data"][20] == "1.0036585365853659" + assert result["data"][21] == "nan" + assert result["data"][22] == "inf" + assert result["data"][23] == "-inf" + assert result["data"][24] is None + assert result["data"][25] == "123.456001" + assert result["data"][26] == "12300.00000" async def test_dry_run_with_connection_url_and_password_with_bracket_should_not_raise_value_error( From 53e0230b694f545bed3ba64387f933af2873c291 Mon Sep 17 00:00:00 2001 From: Jax Liu Date: Fri, 3 Jan 2025 19:16:27 +0800 Subject: [PATCH 5/7] fix test --- ibis-server/tests/routers/v2/connector/test_bigquery.py | 2 +- ibis-server/tests/routers/v2/connector/test_canner.py | 2 +- ibis-server/tests/routers/v2/connector/test_trino.py | 2 +- ibis-server/tests/routers/v3/connector/postgres/test_query.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ibis-server/tests/routers/v2/connector/test_bigquery.py b/ibis-server/tests/routers/v2/connector/test_bigquery.py index 81c7ccf48..054112db1 100644 --- a/ibis-server/tests/routers/v2/connector/test_bigquery.py +++ b/ibis-server/tests/routers/v2/connector/test_bigquery.py @@ -90,7 +90,7 @@ async def test_query(client, manifest_str): 1, 370, "O", - 172799.49, + "172799.49", "1996-01-02", "1_370", "2024-01-01 23:59:59.000000", diff --git a/ibis-server/tests/routers/v2/connector/test_canner.py b/ibis-server/tests/routers/v2/connector/test_canner.py index a1754d5b3..d6fcb141f 100644 --- a/ibis-server/tests/routers/v2/connector/test_canner.py +++ b/ibis-server/tests/routers/v2/connector/test_canner.py @@ -94,7 +94,7 @@ async def test_query(client, manifest_str): 1, 370, "O", - 172799.49, + '172799.49', "1996-01-02", "1_370", "2024-01-01 23:59:59.000000", diff --git a/ibis-server/tests/routers/v2/connector/test_trino.py b/ibis-server/tests/routers/v2/connector/test_trino.py index e33627e8a..25cfb40dd 100644 --- a/ibis-server/tests/routers/v2/connector/test_trino.py +++ b/ibis-server/tests/routers/v2/connector/test_trino.py @@ -111,7 +111,7 @@ async def test_query(client, manifest_str, trino: TrinoContainer): 1, 370, "O", - 172799.49, + "172799.49", "1996-01-02", "1_370", "2024-01-01 23:59:59.000000", diff --git a/ibis-server/tests/routers/v3/connector/postgres/test_query.py b/ibis-server/tests/routers/v3/connector/postgres/test_query.py index d4406ff6e..a37394cbf 100644 --- a/ibis-server/tests/routers/v3/connector/postgres/test_query.py +++ b/ibis-server/tests/routers/v3/connector/postgres/test_query.py @@ -115,7 +115,7 @@ async def test_query(client, manifest_str, connection_info): "2024-01-01 23:59:59.000000 UTC", "2024-01-16 04:00:00.000000 UTC", # utc-5 "2024-07-16 03:00:00.000000 UTC", # utc-4 - 172799.49, + "172799.49", "1_370", 370, "1996-01-02", From d85ef4efcf8697be0039477d5188124670689629 Mon Sep 17 00:00:00 2001 From: Jax Liu Date: Fri, 3 Jan 2025 19:17:57 +0800 Subject: [PATCH 6/7] fix ruff --- ibis-server/tests/routers/v2/connector/test_canner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibis-server/tests/routers/v2/connector/test_canner.py b/ibis-server/tests/routers/v2/connector/test_canner.py index d6fcb141f..9eb368382 100644 --- a/ibis-server/tests/routers/v2/connector/test_canner.py +++ b/ibis-server/tests/routers/v2/connector/test_canner.py @@ -94,7 +94,7 @@ async def test_query(client, manifest_str): 1, 370, "O", - '172799.49', + "172799.49", "1996-01-02", "1_370", "2024-01-01 23:59:59.000000", From 422fac8d86eabd78f9c7efecc042bb4885c5b277 Mon Sep 17 00:00:00 2001 From: Jax Liu Date: Mon, 6 Jan 2025 09:55:07 +0800 Subject: [PATCH 7/7] fix test --- .../routers/v2/connector/test_postgres.py | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/ibis-server/tests/routers/v2/connector/test_postgres.py b/ibis-server/tests/routers/v2/connector/test_postgres.py index 09c36715b..53001867f 100644 --- a/ibis-server/tests/routers/v2/connector/test_postgres.py +++ b/ibis-server/tests/routers/v2/connector/test_postgres.py @@ -256,33 +256,33 @@ async def test_format_floating(client, manifest_str, postgres): assert response.status_code == 200 result = response.json() - assert result["data"][0] == "1.23E-7" - assert result["data"][1] == "1.23E+4" - assert result["data"][2] == "-0.00456" - assert result["data"][3] == "7.89" - assert result["data"][4] == "0" - assert result["data"][5] == "123.456" - assert result["data"][6] == "-123.456" - assert result["data"][7] == "0.0000123" - assert result["data"][8] == "123" - assert result["data"][9] == "0" - assert result["data"][10] == 0 - assert result["data"][11] == "0" - assert result["data"][12] == -1 - assert result["data"][13] == 9999999999 - assert result["data"][14] == "1.7976931348623157E+308" - assert result["data"][15] == "2.2250738585072014E-308" - assert result["data"][16] == "-1.7976931348623157E+308" - assert result["data"][17] == "12304.56" - assert result["data"][18] == "-123.450123" - assert result["data"][19] == "0.000123" - assert result["data"][20] == "1.0036585365853659" - assert result["data"][21] == "nan" - assert result["data"][22] == "inf" - assert result["data"][23] == "-inf" - assert result["data"][24] is None - assert result["data"][25] == "123.456001" - assert result["data"][26] == "12300.00000" + assert result["data"][0][0] == "1.23E-7" + assert result["data"][0][1] == "1.23E+4" + assert result["data"][0][2] == "-0.00456" + assert result["data"][0][3] == "7.89" + assert result["data"][0][4] == "0" + assert result["data"][0][5] == "123.456" + assert result["data"][0][6] == "-123.456" + assert result["data"][0][7] == "0.0000123" + assert result["data"][0][8] == "123" + assert result["data"][0][9] == "0" + assert result["data"][0][10] == 0 + assert result["data"][0][11] == "0" + assert result["data"][0][12] == -1 + assert result["data"][0][13] == 9999999999 + assert result["data"][0][14] == "1.7976931348623157E+308" + assert result["data"][0][15] == "2.2250738585072014E-308" + assert result["data"][0][16] == "-1.7976931348623157E+308" + assert result["data"][0][17] == "12304.56" + assert result["data"][0][18] == "-123.450123" + assert result["data"][0][19] == "0.000123" + assert result["data"][0][20] == "1.0036585365853659" + assert result["data"][0][21] == "nan" + assert result["data"][0][22] == "inf" + assert result["data"][0][23] == "-inf" + assert result["data"][0][24] is None + assert result["data"][0][25] == "123.456001" + assert result["data"][0][26] == "12300.00000" async def test_dry_run_with_connection_url_and_password_with_bracket_should_not_raise_value_error(