diff --git a/ibis-server/app/model/connector.py b/ibis-server/app/model/connector.py index ad0560c16..5b3e54852 100644 --- a/ibis-server/app/model/connector.py +++ b/ibis-server/app/model/connector.py @@ -1,14 +1,17 @@ from json import loads +import pandas as pd + from app.mdl.rewriter import rewrite from app.model.data_source import DataSource, ConnectionInfo class Connector: - def __init__(self, data_source: DataSource, connection_info: ConnectionInfo, manifest_str: str): + def __init__(self, data_source: DataSource, connection_info: ConnectionInfo, manifest_str: str, column_dtypes: dict[str, str]): self.data_source = data_source self.connection = self.data_source.get_connection(connection_info) self.manifest_str = manifest_str + self.column_dtypes = column_dtypes def query(self, sql) -> dict: rewritten_sql = rewrite(self.manifest_str, sql) @@ -21,13 +24,26 @@ def dry_run(self, sql) -> None: except Exception as e: raise QueryDryRunError(f'Exception: {type(e)}, message: {str(e)}') - @staticmethod - def _to_json(df): + def _to_json(self, df): + if self.column_dtypes: + self._to_specific_types(df, self.column_dtypes) json_obj = loads(df.to_json(orient='split')) del json_obj['index'] json_obj['dtypes'] = df.dtypes.apply(lambda x: x.name).to_dict() return json_obj + def _to_specific_types(self, df: pd.DataFrame, column_dtypes: dict[str, str]): + for column, dtype in column_dtypes.items(): + if dtype == 'datetime64': + df[column] = self._to_datetime_and_format(df[column]) + else: + df[column] = df[column].astype(dtype) + + @staticmethod + def _to_datetime_and_format(series: pd.Series) -> pd.Series: + series = pd.to_datetime(series, errors='coerce') + return series.apply(lambda d: d.strftime('%Y-%m-%d %H:%M:%S.%f' + (' %Z' if series.dt.tz is not None else '')) if not pd.isnull(d) else d) + class QueryDryRunError(Exception): pass diff --git a/ibis-server/app/model/dto.py b/ibis-server/app/model/dto.py index f93561061..e7469928f 100644 --- a/ibis-server/app/model/dto.py +++ b/ibis-server/app/model/dto.py @@ -13,6 +13,7 @@ class IbisDTO(BaseModel): sql: str manifest_str: str = Field(alias="manifestStr", description="Base64 manifest") + column_dtypes: dict[str, str] | None = Field(alias="columnDtypes", description="If this field is set, it will forcibly convert the type.", default=None) class PostgresDTO(IbisDTO): diff --git a/ibis-server/app/routers/ibis/bigquery.py b/ibis-server/app/routers/ibis/bigquery.py index c0026ee78..0c97bc756 100644 --- a/ibis-server/app/routers/ibis/bigquery.py +++ b/ibis-server/app/routers/ibis/bigquery.py @@ -16,7 +16,7 @@ @router.post("/query") @log_dto def query(dto: BigQueryDTO, dry_run: Annotated[bool, Query(alias="dryRun")] = False) -> Response: - connector = Connector(data_source, dto.connection_info, dto.manifest_str) + connector = Connector(data_source, dto.connection_info, dto.manifest_str, dto.column_dtypes) if dry_run: connector.dry_run(dto.sql) return Response(status_code=204) diff --git a/ibis-server/app/routers/ibis/postgres.py b/ibis-server/app/routers/ibis/postgres.py index 3f29533ab..3c73ebe97 100644 --- a/ibis-server/app/routers/ibis/postgres.py +++ b/ibis-server/app/routers/ibis/postgres.py @@ -16,7 +16,7 @@ @router.post("/query") @log_dto def query(dto: PostgresDTO, dry_run: Annotated[bool, Query(alias="dryRun")] = False) -> Response: - connector = Connector(data_source, dto.connection_info, dto.manifest_str) + connector = Connector(data_source, dto.connection_info, dto.manifest_str, dto.column_dtypes) if dry_run: connector.dry_run(dto.sql) return Response(status_code=204) diff --git a/ibis-server/app/routers/ibis/snowflake.py b/ibis-server/app/routers/ibis/snowflake.py index 1603e6623..9cc4d793a 100644 --- a/ibis-server/app/routers/ibis/snowflake.py +++ b/ibis-server/app/routers/ibis/snowflake.py @@ -16,7 +16,7 @@ @router.post("/query") @log_dto def query(dto: SnowflakeDTO, dry_run: Annotated[bool, Query(alias="dryRun")] = False) -> Response: - connector = Connector(data_source, dto.connection_info, dto.manifest_str) + connector = Connector(data_source, dto.connection_info, dto.manifest_str, dto.column_dtypes) if dry_run: connector.dry_run(dto.sql) return Response(status_code=204) diff --git a/ibis-server/poetry.lock b/ibis-server/poetry.lock index 6f57712bf..b6bb1d865 100644 --- a/ibis-server/poetry.lock +++ b/ibis-server/poetry.lock @@ -77,13 +77,13 @@ files = [ [[package]] name = "certifi" -version = "2024.2.2" +version = "2024.6.2" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"}, - {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, + {file = "certifi-2024.6.2-py3-none-any.whl", hash = "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"}, + {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"}, ] [[package]] @@ -276,43 +276,43 @@ files = [ [[package]] name = "cryptography" -version = "42.0.7" +version = "42.0.8" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-42.0.7-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:a987f840718078212fdf4504d0fd4c6effe34a7e4740378e59d47696e8dfb477"}, - {file = "cryptography-42.0.7-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:bd13b5e9b543532453de08bcdc3cc7cebec6f9883e886fd20a92f26940fd3e7a"}, - {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a79165431551042cc9d1d90e6145d5d0d3ab0f2d66326c201d9b0e7f5bf43604"}, - {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a47787a5e3649008a1102d3df55424e86606c9bae6fb77ac59afe06d234605f8"}, - {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:02c0eee2d7133bdbbc5e24441258d5d2244beb31da5ed19fbb80315f4bbbff55"}, - {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:5e44507bf8d14b36b8389b226665d597bc0f18ea035d75b4e53c7b1ea84583cc"}, - {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:7f8b25fa616d8b846aef64b15c606bb0828dbc35faf90566eb139aa9cff67af2"}, - {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:93a3209f6bb2b33e725ed08ee0991b92976dfdcf4e8b38646540674fc7508e13"}, - {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e6b8f1881dac458c34778d0a424ae5769de30544fc678eac51c1c8bb2183e9da"}, - {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3de9a45d3b2b7d8088c3fbf1ed4395dfeff79d07842217b38df14ef09ce1d8d7"}, - {file = "cryptography-42.0.7-cp37-abi3-win32.whl", hash = "sha256:789caea816c6704f63f6241a519bfa347f72fbd67ba28d04636b7c6b7da94b0b"}, - {file = "cryptography-42.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:8cb8ce7c3347fcf9446f201dc30e2d5a3c898d009126010cbd1f443f28b52678"}, - {file = "cryptography-42.0.7-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:a3a5ac8b56fe37f3125e5b72b61dcde43283e5370827f5233893d461b7360cd4"}, - {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:779245e13b9a6638df14641d029add5dc17edbef6ec915688f3acb9e720a5858"}, - {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d563795db98b4cd57742a78a288cdbdc9daedac29f2239793071fe114f13785"}, - {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:31adb7d06fe4383226c3e963471f6837742889b3c4caa55aac20ad951bc8ffda"}, - {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:efd0bf5205240182e0f13bcaea41be4fdf5c22c5129fc7ced4a0282ac86998c9"}, - {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a9bc127cdc4ecf87a5ea22a2556cab6c7eda2923f84e4f3cc588e8470ce4e42e"}, - {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:3577d029bc3f4827dd5bf8bf7710cac13527b470bbf1820a3f394adb38ed7d5f"}, - {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2e47577f9b18723fa294b0ea9a17d5e53a227867a0a4904a1a076d1646d45ca1"}, - {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1a58839984d9cb34c855197043eaae2c187d930ca6d644612843b4fe8513c886"}, - {file = "cryptography-42.0.7-cp39-abi3-win32.whl", hash = "sha256:e6b79d0adb01aae87e8a44c2b64bc3f3fe59515280e00fb6d57a7267a2583cda"}, - {file = "cryptography-42.0.7-cp39-abi3-win_amd64.whl", hash = "sha256:16268d46086bb8ad5bf0a2b5544d8a9ed87a0e33f5e77dd3c3301e63d941a83b"}, - {file = "cryptography-42.0.7-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:2954fccea107026512b15afb4aa664a5640cd0af630e2ee3962f2602693f0c82"}, - {file = "cryptography-42.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:362e7197754c231797ec45ee081f3088a27a47c6c01eff2ac83f60f85a50fe60"}, - {file = "cryptography-42.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4f698edacf9c9e0371112792558d2f705b5645076cc0aaae02f816a0171770fd"}, - {file = "cryptography-42.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5482e789294854c28237bba77c4c83be698be740e31a3ae5e879ee5444166582"}, - {file = "cryptography-42.0.7-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e9b2a6309f14c0497f348d08a065d52f3020656f675819fc405fb63bbcd26562"}, - {file = "cryptography-42.0.7-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d8e3098721b84392ee45af2dd554c947c32cc52f862b6a3ae982dbb90f577f14"}, - {file = "cryptography-42.0.7-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c65f96dad14f8528a447414125e1fc8feb2ad5a272b8f68477abbcc1ea7d94b9"}, - {file = "cryptography-42.0.7-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:36017400817987670037fbb0324d71489b6ead6231c9604f8fc1f7d008087c68"}, - {file = "cryptography-42.0.7.tar.gz", hash = "sha256:ecbfbc00bf55888edda9868a4cf927205de8499e7fabe6c050322298382953f2"}, + {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e"}, + {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7"}, + {file = "cryptography-42.0.8-cp37-abi3-win32.whl", hash = "sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2"}, + {file = "cryptography-42.0.8-cp37-abi3-win_amd64.whl", hash = "sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba"}, + {file = "cryptography-42.0.8-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14"}, + {file = "cryptography-42.0.8-cp39-abi3-win32.whl", hash = "sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c"}, + {file = "cryptography-42.0.8-cp39-abi3-win_amd64.whl", hash = "sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad"}, + {file = "cryptography-42.0.8.tar.gz", hash = "sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2"}, ] [package.dependencies] @@ -530,13 +530,13 @@ tool = ["click (>=6.0.0)"] [[package]] name = "google-cloud-bigquery" -version = "3.23.1" +version = "3.24.0" description = "Google BigQuery API client library" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-bigquery-3.23.1.tar.gz", hash = "sha256:4b4597f9291b42102c9667d3b4528f801d4c8f24ef2b12dd1ecb881273330955"}, - {file = "google_cloud_bigquery-3.23.1-py2.py3-none-any.whl", hash = "sha256:9fb72884fdbec9c4643cea6b7f21e1ecf3eb61d5305f87493d271dc801647a9e"}, + {file = "google-cloud-bigquery-3.24.0.tar.gz", hash = "sha256:e95e6f6e0aa32e6c453d44e2b3298931fdd7947c309ea329a31b6ff1f939e17e"}, + {file = "google_cloud_bigquery-3.24.0-py2.py3-none-any.whl", hash = "sha256:bc08323ce99dee4e811b7c3d0cde8929f5bf0b1aeaed6bcd75fc89796dd87652"}, ] [package.dependencies] @@ -699,17 +699,17 @@ requests = ["requests (>=2.18.0,<3.0.0dev)"] [[package]] name = "googleapis-common-protos" -version = "1.63.0" +version = "1.63.1" description = "Common protobufs used in Google APIs" optional = false python-versions = ">=3.7" files = [ - {file = "googleapis-common-protos-1.63.0.tar.gz", hash = "sha256:17ad01b11d5f1d0171c06d3ba5c04c54474e883b66b949722b4938ee2694ef4e"}, - {file = "googleapis_common_protos-1.63.0-py2.py3-none-any.whl", hash = "sha256:ae45f75702f7c08b541f750854a678bd8f534a1a6bace6afe975f1d0a82d6632"}, + {file = "googleapis-common-protos-1.63.1.tar.gz", hash = "sha256:c6442f7a0a6b2a80369457d79e6672bb7dcbaab88e0848302497e3ec80780a6a"}, + {file = "googleapis_common_protos-1.63.1-py2.py3-none-any.whl", hash = "sha256:0e1c2cdfcbc354b76e4a211a35ea35d6926a835cba1377073c4861db904a1877"}, ] [package.dependencies] -protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0.dev0" [package.extras] grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] @@ -787,61 +787,61 @@ test = ["objgraph", "psutil"] [[package]] name = "grpcio" -version = "1.64.0" +version = "1.64.1" description = "HTTP/2-based RPC framework" optional = false python-versions = ">=3.8" files = [ - {file = "grpcio-1.64.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:3b09c3d9de95461214a11d82cc0e6a46a6f4e1f91834b50782f932895215e5db"}, - {file = "grpcio-1.64.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:7e013428ab472892830287dd082b7d129f4d8afef49227a28223a77337555eaa"}, - {file = "grpcio-1.64.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:02cc9cc3f816d30f7993d0d408043b4a7d6a02346d251694d8ab1f78cc723e7e"}, - {file = "grpcio-1.64.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f5de082d936e0208ce8db9095821361dfa97af8767a6607ae71425ac8ace15c"}, - {file = "grpcio-1.64.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7b7bf346391dffa182fba42506adf3a84f4a718a05e445b37824136047686a1"}, - {file = "grpcio-1.64.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b2cbdfba18408389a1371f8c2af1659119e1831e5ed24c240cae9e27b4abc38d"}, - {file = "grpcio-1.64.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:aca4f15427d2df592e0c8f3d38847e25135e4092d7f70f02452c0e90d6a02d6d"}, - {file = "grpcio-1.64.0-cp310-cp310-win32.whl", hash = "sha256:7c1f5b2298244472bcda49b599be04579f26425af0fd80d3f2eb5fd8bc84d106"}, - {file = "grpcio-1.64.0-cp310-cp310-win_amd64.whl", hash = "sha256:73f84f9e5985a532e47880b3924867de16fa1aa513fff9b26106220c253c70c5"}, - {file = "grpcio-1.64.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:2a18090371d138a57714ee9bffd6c9c9cb2e02ce42c681aac093ae1e7189ed21"}, - {file = "grpcio-1.64.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:59c68df3a934a586c3473d15956d23a618b8f05b5e7a3a904d40300e9c69cbf0"}, - {file = "grpcio-1.64.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:b52e1ec7185512103dd47d41cf34ea78e7a7361ba460187ddd2416b480e0938c"}, - {file = "grpcio-1.64.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8d598b5d5e2c9115d7fb7e2cb5508d14286af506a75950762aa1372d60e41851"}, - {file = "grpcio-1.64.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01615bbcae6875eee8091e6b9414072f4e4b00d8b7e141f89635bdae7cf784e5"}, - {file = "grpcio-1.64.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:0b2dfe6dcace264807d9123d483d4c43274e3f8c39f90ff51de538245d7a4145"}, - {file = "grpcio-1.64.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7f17572dc9acd5e6dfd3014d10c0b533e9f79cd9517fc10b0225746f4c24b58e"}, - {file = "grpcio-1.64.0-cp311-cp311-win32.whl", hash = "sha256:6ec5ed15b4ffe56e2c6bc76af45e6b591c9be0224b3fb090adfb205c9012367d"}, - {file = "grpcio-1.64.0-cp311-cp311-win_amd64.whl", hash = "sha256:597191370951b477b7a1441e1aaa5cacebeb46a3b0bd240ec3bb2f28298c7553"}, - {file = "grpcio-1.64.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:1ce4cd5a61d4532651079e7aae0fedf9a80e613eed895d5b9743e66b52d15812"}, - {file = "grpcio-1.64.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:650a8150a9b288f40d5b7c1d5400cc11724eae50bd1f501a66e1ea949173649b"}, - {file = "grpcio-1.64.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:8de0399b983f8676a7ccfdd45e5b2caec74a7e3cc576c6b1eecf3b3680deda5e"}, - {file = "grpcio-1.64.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:46b8b43ba6a2a8f3103f103f97996cad507bcfd72359af6516363c48793d5a7b"}, - {file = "grpcio-1.64.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a54362f03d4dcfae63be455d0a7d4c1403673498b92c6bfe22157d935b57c7a9"}, - {file = "grpcio-1.64.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:1f8ea18b928e539046bb5f9c124d717fbf00cc4b2d960ae0b8468562846f5aa1"}, - {file = "grpcio-1.64.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c56c91bd2923ddb6e7ed28ebb66d15633b03e0df22206f22dfcdde08047e0a48"}, - {file = "grpcio-1.64.0-cp312-cp312-win32.whl", hash = "sha256:874c741c8a66f0834f653a69e7e64b4e67fcd4a8d40296919b93bab2ccc780ba"}, - {file = "grpcio-1.64.0-cp312-cp312-win_amd64.whl", hash = "sha256:0da1d921f8e4bcee307aeef6c7095eb26e617c471f8cb1c454fd389c5c296d1e"}, - {file = "grpcio-1.64.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:c46fb6bfca17bfc49f011eb53416e61472fa96caa0979b4329176bdd38cbbf2a"}, - {file = "grpcio-1.64.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:3d2004e85cf5213995d09408501f82c8534700d2babeb81dfdba2a3bff0bb396"}, - {file = "grpcio-1.64.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:6d5541eb460d73a07418524fb64dcfe0adfbcd32e2dac0f8f90ce5b9dd6c046c"}, - {file = "grpcio-1.64.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f279ad72dd7d64412e10f2443f9f34872a938c67387863c4cd2fb837f53e7d2"}, - {file = "grpcio-1.64.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85fda90b81da25993aa47fae66cae747b921f8f6777550895fb62375b776a231"}, - {file = "grpcio-1.64.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a053584079b793a54bece4a7d1d1b5c0645bdbee729215cd433703dc2532f72b"}, - {file = "grpcio-1.64.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:579dd9fb11bc73f0de061cab5f8b2def21480fd99eb3743ed041ad6a1913ee2f"}, - {file = "grpcio-1.64.0-cp38-cp38-win32.whl", hash = "sha256:23b6887bb21d77649d022fa1859e05853fdc2e60682fd86c3db652a555a282e0"}, - {file = "grpcio-1.64.0-cp38-cp38-win_amd64.whl", hash = "sha256:753cb58683ba0c545306f4e17dabf468d29cb6f6b11832e1e432160bb3f8403c"}, - {file = "grpcio-1.64.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:2186d76a7e383e1466e0ea2b0febc343ffeae13928c63c6ec6826533c2d69590"}, - {file = "grpcio-1.64.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0f30596cdcbed3c98024fb4f1d91745146385b3f9fd10c9f2270cbfe2ed7ed91"}, - {file = "grpcio-1.64.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:d9171f025a196f5bcfec7e8e7ffb7c3535f7d60aecd3503f9e250296c7cfc150"}, - {file = "grpcio-1.64.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cf4c8daed18ae2be2f1fc7d613a76ee2a2e28fdf2412d5c128be23144d28283d"}, - {file = "grpcio-1.64.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3550493ac1d23198d46dc9c9b24b411cef613798dc31160c7138568ec26bc9b4"}, - {file = "grpcio-1.64.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:3161a8f8bb38077a6470508c1a7301cd54301c53b8a34bb83e3c9764874ecabd"}, - {file = "grpcio-1.64.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2e8fabe2cc57a369638ab1ad8e6043721014fdf9a13baa7c0e35995d3a4a7618"}, - {file = "grpcio-1.64.0-cp39-cp39-win32.whl", hash = "sha256:31890b24d47b62cc27da49a462efe3d02f3c120edb0e6c46dcc0025506acf004"}, - {file = "grpcio-1.64.0-cp39-cp39-win_amd64.whl", hash = "sha256:5a56797dea8c02e7d3a85dfea879f286175cf4d14fbd9ab3ef2477277b927baa"}, - {file = "grpcio-1.64.0.tar.gz", hash = "sha256:257baf07f53a571c215eebe9679c3058a313fd1d1f7c4eede5a8660108c52d9c"}, + {file = "grpcio-1.64.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:55697ecec192bc3f2f3cc13a295ab670f51de29884ca9ae6cd6247df55df2502"}, + {file = "grpcio-1.64.1-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:3b64ae304c175671efdaa7ec9ae2cc36996b681eb63ca39c464958396697daff"}, + {file = "grpcio-1.64.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:bac71b4b28bc9af61efcdc7630b166440bbfbaa80940c9a697271b5e1dabbc61"}, + {file = "grpcio-1.64.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c024ffc22d6dc59000faf8ad781696d81e8e38f4078cb0f2630b4a3cf231a90"}, + {file = "grpcio-1.64.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7cd5c1325f6808b8ae31657d281aadb2a51ac11ab081ae335f4f7fc44c1721d"}, + {file = "grpcio-1.64.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:0a2813093ddb27418a4c99f9b1c223fab0b053157176a64cc9db0f4557b69bd9"}, + {file = "grpcio-1.64.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2981c7365a9353f9b5c864595c510c983251b1ab403e05b1ccc70a3d9541a73b"}, + {file = "grpcio-1.64.1-cp310-cp310-win32.whl", hash = "sha256:1262402af5a511c245c3ae918167eca57342c72320dffae5d9b51840c4b2f86d"}, + {file = "grpcio-1.64.1-cp310-cp310-win_amd64.whl", hash = "sha256:19264fc964576ddb065368cae953f8d0514ecc6cb3da8903766d9fb9d4554c33"}, + {file = "grpcio-1.64.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:58b1041e7c870bb30ee41d3090cbd6f0851f30ae4eb68228955d973d3efa2e61"}, + {file = "grpcio-1.64.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:bbc5b1d78a7822b0a84c6f8917faa986c1a744e65d762ef6d8be9d75677af2ca"}, + {file = "grpcio-1.64.1-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:5841dd1f284bd1b3d8a6eca3a7f062b06f1eec09b184397e1d1d43447e89a7ae"}, + {file = "grpcio-1.64.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8caee47e970b92b3dd948371230fcceb80d3f2277b3bf7fbd7c0564e7d39068e"}, + {file = "grpcio-1.64.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73819689c169417a4f978e562d24f2def2be75739c4bed1992435d007819da1b"}, + {file = "grpcio-1.64.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6503b64c8b2dfad299749cad1b595c650c91e5b2c8a1b775380fcf8d2cbba1e9"}, + {file = "grpcio-1.64.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1de403fc1305fd96cfa75e83be3dee8538f2413a6b1685b8452301c7ba33c294"}, + {file = "grpcio-1.64.1-cp311-cp311-win32.whl", hash = "sha256:d4d29cc612e1332237877dfa7fe687157973aab1d63bd0f84cf06692f04c0367"}, + {file = "grpcio-1.64.1-cp311-cp311-win_amd64.whl", hash = "sha256:5e56462b05a6f860b72f0fa50dca06d5b26543a4e88d0396259a07dc30f4e5aa"}, + {file = "grpcio-1.64.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:4657d24c8063e6095f850b68f2d1ba3b39f2b287a38242dcabc166453e950c59"}, + {file = "grpcio-1.64.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:62b4e6eb7bf901719fce0ca83e3ed474ae5022bb3827b0a501e056458c51c0a1"}, + {file = "grpcio-1.64.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:ee73a2f5ca4ba44fa33b4d7d2c71e2c8a9e9f78d53f6507ad68e7d2ad5f64a22"}, + {file = "grpcio-1.64.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:198908f9b22e2672a998870355e226a725aeab327ac4e6ff3a1399792ece4762"}, + {file = "grpcio-1.64.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39b9d0acaa8d835a6566c640f48b50054f422d03e77e49716d4c4e8e279665a1"}, + {file = "grpcio-1.64.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:5e42634a989c3aa6049f132266faf6b949ec2a6f7d302dbb5c15395b77d757eb"}, + {file = "grpcio-1.64.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b1a82e0b9b3022799c336e1fc0f6210adc019ae84efb7321d668129d28ee1efb"}, + {file = "grpcio-1.64.1-cp312-cp312-win32.whl", hash = "sha256:55260032b95c49bee69a423c2f5365baa9369d2f7d233e933564d8a47b893027"}, + {file = "grpcio-1.64.1-cp312-cp312-win_amd64.whl", hash = "sha256:c1a786ac592b47573a5bb7e35665c08064a5d77ab88a076eec11f8ae86b3e3f6"}, + {file = "grpcio-1.64.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:a011ac6c03cfe162ff2b727bcb530567826cec85eb8d4ad2bfb4bd023287a52d"}, + {file = "grpcio-1.64.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:4d6dab6124225496010bd22690f2d9bd35c7cbb267b3f14e7a3eb05c911325d4"}, + {file = "grpcio-1.64.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:a5e771d0252e871ce194d0fdcafd13971f1aae0ddacc5f25615030d5df55c3a2"}, + {file = "grpcio-1.64.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2c3c1b90ab93fed424e454e93c0ed0b9d552bdf1b0929712b094f5ecfe7a23ad"}, + {file = "grpcio-1.64.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20405cb8b13fd779135df23fabadc53b86522d0f1cba8cca0e87968587f50650"}, + {file = "grpcio-1.64.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0cc79c982ccb2feec8aad0e8fb0d168bcbca85bc77b080d0d3c5f2f15c24ea8f"}, + {file = "grpcio-1.64.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:a3a035c37ce7565b8f4f35ff683a4db34d24e53dc487e47438e434eb3f701b2a"}, + {file = "grpcio-1.64.1-cp38-cp38-win32.whl", hash = "sha256:1257b76748612aca0f89beec7fa0615727fd6f2a1ad580a9638816a4b2eb18fd"}, + {file = "grpcio-1.64.1-cp38-cp38-win_amd64.whl", hash = "sha256:0a12ddb1678ebc6a84ec6b0487feac020ee2b1659cbe69b80f06dbffdb249122"}, + {file = "grpcio-1.64.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:75dbbf415026d2862192fe1b28d71f209e2fd87079d98470db90bebe57b33179"}, + {file = "grpcio-1.64.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e3d9f8d1221baa0ced7ec7322a981e28deb23749c76eeeb3d33e18b72935ab62"}, + {file = "grpcio-1.64.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:5f8b75f64d5d324c565b263c67dbe4f0af595635bbdd93bb1a88189fc62ed2e5"}, + {file = "grpcio-1.64.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c84ad903d0d94311a2b7eea608da163dace97c5fe9412ea311e72c3684925602"}, + {file = "grpcio-1.64.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:940e3ec884520155f68a3b712d045e077d61c520a195d1a5932c531f11883489"}, + {file = "grpcio-1.64.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f10193c69fc9d3d726e83bbf0f3d316f1847c3071c8c93d8090cf5f326b14309"}, + {file = "grpcio-1.64.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ac15b6c2c80a4d1338b04d42a02d376a53395ddf0ec9ab157cbaf44191f3ffdd"}, + {file = "grpcio-1.64.1-cp39-cp39-win32.whl", hash = "sha256:03b43d0ccf99c557ec671c7dede64f023c7da9bb632ac65dbc57f166e4970040"}, + {file = "grpcio-1.64.1-cp39-cp39-win_amd64.whl", hash = "sha256:ed6091fa0adcc7e4ff944090cf203a52da35c37a130efa564ded02b7aff63bcd"}, + {file = "grpcio-1.64.1.tar.gz", hash = "sha256:8d51dd1c59d5fa0f34266b80a3805ec29a1f26425c2a54736133f6d87fc4968a"}, ] [package.extras] -protobuf = ["grpcio-tools (>=1.64.0)"] +protobuf = ["grpcio-tools (>=1.64.1)"] [[package]] name = "grpcio-status" @@ -2278,13 +2278,13 @@ typing-extensions = ">=3.7.4.3" [[package]] name = "typing-extensions" -version = "4.12.0" +version = "4.12.1" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.12.0-py3-none-any.whl", hash = "sha256:b349c66bea9016ac22978d800cfff206d5f9816951f12a7d0ec5578b0a819594"}, - {file = "typing_extensions-4.12.0.tar.gz", hash = "sha256:8cbcdc8606ebcb0d95453ad7dc5065e6237b6aa230a31e81d0f440c30fed5fd8"}, + {file = "typing_extensions-4.12.1-py3-none-any.whl", hash = "sha256:6024b58b69089e5a89c347397254e35f1bf02a907728ec7fee9bf0fe837d203a"}, + {file = "typing_extensions-4.12.1.tar.gz", hash = "sha256:915f5e35ff76f56588223f15fdd5938f9a1cf9195c0de25130c627e4d597f6d1"}, ] [[package]] @@ -2721,4 +2721,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = ">=3.11,<3.12" -content-hash = "008b5191943af4d37f58bfe72bbd9dbeb38a51e6ef511e18da9fa214bcd66bad" +content-hash = "359c8e6fcb5fd2d1b8caa746737dfdeb9d7725e2602008e048a5c4396ca0e406" diff --git a/ibis-server/pyproject.toml b/ibis-server/pyproject.toml index ac9021d2f..2eeb6b627 100644 --- a/ibis-server/pyproject.toml +++ b/ibis-server/pyproject.toml @@ -16,12 +16,12 @@ google-auth = "2.29.0" httpx = "0.27.0" python-dotenv = "1.0.1" orjson = "3.10.3" +pandas = "2.2.2" [tool.poetry.group.dev.dependencies] pytest = "8.2.0" testcontainers = {extras = ["postgres"], version = "4.5.0"} sqlalchemy = "2.0.30" -pandas = "2.2.2" [tool.pytest.ini_options] addopts = "--strict-markers" diff --git a/ibis-server/tests/routers/ibis/test_bigquery.py b/ibis-server/tests/routers/ibis/test_bigquery.py index dcf36becb..ab5d1fe01 100644 --- a/ibis-server/tests/routers/ibis/test_bigquery.py +++ b/ibis-server/tests/routers/ibis/test_bigquery.py @@ -1,3 +1,6 @@ +import base64 + +import orjson import pytest from fastapi.testclient import TestClient @@ -8,38 +11,38 @@ @pytest.mark.bigquery class TestBigquery: + manifest = { + "catalog": "my_catalog", + "schema": "my_schema", + "models": [ + { + "name": "Orders", + "refSql": "select * from tpch_tiny.orders", + "columns": [ + {"name": "orderkey", "expression": "o_orderkey", "type": "integer"}, + {"name": "custkey", "expression": "o_custkey", "type": "integer"}, + {"name": "orderstatus", "expression": "o_orderstatus", "type": "varchar"}, + {"name": "totalprice", "expression": "o_totalprice", "type": "float"}, + {"name": "orderdate", "expression": "o_orderdate", "type": "date"}, + {"name": "order_cust_key", "expression": "concat(o_orderkey, '_', o_custkey)", "type": "varchar"}, + {"name": "timestamp", "expression": "cast('2024-01-01T23:59:59' as timestamp)", "type": "timestamp"}, + {"name": "timestamptz", "expression": "cast('2024-01-01T23:59:59' as timestamp with time zone)", "type": "timestamp"} + ], + "primaryKey": "orderkey" + }, + { + "name": "Customer", + "refSql": "select * from tpch_tiny.customer", + "columns": [ + {"name": "custkey", "expression": "c_custkey", "type": "integer"}, + {"name": "name", "expression": "c_name", "type": "varchar"} + ], + "primaryKey": "custkey" + } + ] + } - @pytest.fixture() - def manifest_str(self) -> str: - import base64 - import orjson - - manifest = { - "catalog": "my_catalog", - "schema": "my_schema", - "models": [ - { - "name": "Orders", - "properties": {}, - "refSql": "select * from tpch_tiny.orders", - "columns": [ - {"name": "orderkey", "expression": "o_orderkey", "type": "integer"}, - {"name": "custkey", "type": "integer", "expression": "o_custkey"} - ], - "primaryKey": "orderkey" - }, - { - "name": "Customer", - "refSql": "select * from tpch_tiny.customer", - "columns": [ - {"name": "custkey", "expression": "c_custkey", "type": "integer"}, - {"name": "name", "expression": "c_name", "type": "varchar"} - ], - "primaryKey": "custkey" - } - ] - } - return base64.b64encode(orjson.dumps(manifest)).decode('utf-8') + manifest_str = base64.b64encode(orjson.dumps(manifest)).decode('utf-8') @staticmethod def get_connection_info(): @@ -50,22 +53,63 @@ def get_connection_info(): "credentials": os.getenv("TEST_BIG_QUERY_CREDENTIALS_BASE64_JSON") } - def test_query(self, manifest_str: str): + def test_query(self): connection_info = self.get_connection_info() response = client.post( url="/v2/ibis/bigquery/query", json={ "connectionInfo": connection_info, - "manifestStr": manifest_str, - "sql": 'SELECT * FROM "Orders" LIMIT 1' + "manifestStr": self.manifest_str, + "sql": 'SELECT * FROM "Orders" ORDER BY orderkey LIMIT 1' + } + ) + assert response.status_code == 200 + result = response.json() + assert len(result['columns']) == len(self.manifest['models'][0]['columns']) + assert len(result['data']) == 1 + assert result['data'][0] == [1, 370, 'O', 172799.49, 820540800000, '1_370', 1704153599000, 1704153599000] + assert result['dtypes'] == { + 'orderkey': 'int64', + 'custkey': 'int64', + 'orderstatus': 'object', + 'totalprice': 'float64', + 'orderdate': 'object', + 'order_cust_key': 'object', + 'timestamp': 'datetime64[ns]', + 'timestamptz': 'datetime64[ns, UTC]' + } + + def test_query_with_column_dtypes(self): + connection_info = self.get_connection_info() + response = client.post( + url="/v2/ibis/bigquery/query", + json={ + "connectionInfo": connection_info, + "manifestStr": self.manifest_str, + "sql": 'SELECT * FROM "Orders" ORDER BY orderkey LIMIT 1', + "columnDtypes": { + "totalprice": "float", + "orderdate": "datetime64", + "timestamp": "datetime64", + "timestamptz": "datetime64" + } } ) assert response.status_code == 200 result = response.json() - assert len(result['columns']) == 2 + assert len(result['columns']) == len(self.manifest['models'][0]['columns']) assert len(result['data']) == 1 - assert result['data'][0][0] is not None - assert result['dtypes'] is not None + assert result['data'][0] == [1, 370, 'O', 172799.49, '1996-01-02 00:00:00.000000', '1_370', '2024-01-01 23:59:59.000000', '2024-01-01 23:59:59.000000 UTC'] + assert result['dtypes'] == { + 'orderkey': 'int64', + 'custkey': 'int64', + 'orderstatus': 'object', + 'totalprice': 'float64', + 'orderdate': 'object', + 'order_cust_key': 'object', + 'timestamp': 'object', + 'timestamptz': 'object' + } def test_query_without_manifest(self): connection_info = self.get_connection_info() @@ -83,13 +127,13 @@ def test_query_without_manifest(self): assert result['detail'][0]['loc'] == ['body', 'manifestStr'] assert result['detail'][0]['msg'] == 'Field required' - def test_query_without_sql(self, manifest_str: str): + def test_query_without_sql(self): connection_info = self.get_connection_info() response = client.post( url="/v2/ibis/bigquery/query", json={ "connectionInfo": connection_info, - "manifestStr": manifest_str + "manifestStr": self.manifest_str } ) assert response.status_code == 422 @@ -99,11 +143,11 @@ def test_query_without_sql(self, manifest_str: str): assert result['detail'][0]['loc'] == ['body', 'sql'] assert result['detail'][0]['msg'] == 'Field required' - def test_query_without_connection_info(self, manifest_str: str): + def test_query_without_connection_info(self): response = client.post( url="/v2/ibis/bigquery/query", json={ - "manifestStr": manifest_str, + "manifestStr": self.manifest_str, "sql": 'SELECT * FROM "Orders" LIMIT 1' } ) @@ -114,27 +158,27 @@ def test_query_without_connection_info(self, manifest_str: str): assert result['detail'][0]['loc'] == ['body', 'connectionInfo'] assert result['detail'][0]['msg'] == 'Field required' - def test_query_with_dry_run(self, manifest_str: str): + def test_query_with_dry_run(self): connection_info = self.get_connection_info() response = client.post( url="/v2/ibis/bigquery/query", params={"dryRun": True}, json={ "connectionInfo": connection_info, - "manifestStr": manifest_str, + "manifestStr": self.manifest_str, "sql": 'SELECT * FROM "Orders" LIMIT 1' } ) assert response.status_code == 204 - def test_query_with_dry_run_and_invalid_sql(self, manifest_str: str): + def test_query_with_dry_run_and_invalid_sql(self): connection_info = self.get_connection_info() response = client.post( url="/v2/ibis/bigquery/query", params={"dryRun": True}, json={ "connectionInfo": connection_info, - "manifestStr": manifest_str, + "manifestStr": self.manifest_str, "sql": 'SELECT * FROM X' } ) diff --git a/ibis-server/tests/routers/ibis/test_postgres.py b/ibis-server/tests/routers/ibis/test_postgres.py index d0a2669df..885efd808 100644 --- a/ibis-server/tests/routers/ibis/test_postgres.py +++ b/ibis-server/tests/routers/ibis/test_postgres.py @@ -1,3 +1,6 @@ +import base64 + +import orjson import pytest from fastapi.testclient import TestClient from testcontainers.postgres import PostgresContainer @@ -33,38 +36,38 @@ def stop_pg(): @pytest.mark.postgres class TestPostgres: + manifest = { + "catalog": "my_catalog", + "schema": "my_schema", + "models": [ + { + "name": "Orders", + "refSql": "select * from public.orders", + "columns": [ + {"name": "orderkey", "expression": "o_orderkey", "type": "integer"}, + {"name": "custkey", "expression": "o_custkey", "type": "integer"}, + {"name": "orderstatus", "expression": "o_orderstatus", "type": "varchar"}, + {"name": "totalprice", "expression": "o_totalprice", "type": "float"}, + {"name": "orderdate", "expression": "o_orderdate", "type": "date"}, + {"name": "order_cust_key", "expression": "concat(o_orderkey, '_', o_custkey)", "type": "varchar"}, + {"name": "timestamp", "expression": "cast('2024-01-01T23:59:59' as timestamp)", "type": "timestamp"}, + {"name": "timestamptz", "expression": "cast('2024-01-01T23:59:59' as timestamp with time zone)", "type": "timestamp"} + ], + "primaryKey": "orderkey" + }, + { + "name": "Customer", + "refSql": "select * from public.customer", + "columns": [ + {"name": "custkey", "expression": "c_custkey", "type": "integer"}, + {"name": "name", "expression": "c_name", "type": "varchar"} + ], + "primaryKey": "custkey" + } + ] + } - @pytest.fixture() - def manifest_str(self) -> str: - import base64 - import orjson - - manifest = { - "catalog": "my_catalog", - "schema": "my_schema", - "models": [ - { - "name": "Orders", - "properties": {}, - "refSql": "select * from public.orders", - "columns": [ - {"name": "orderkey", "expression": "o_orderkey", "type": "integer"}, - {"name": "custkey", "type": "integer", "expression": "o_custkey"} - ], - "primaryKey": "orderkey" - }, - { - "name": "Customer", - "refSql": "select * from public.customer", - "columns": [ - {"name": "custkey", "expression": "c_custkey", "type": "integer"}, - {"name": "name", "expression": "c_name", "type": "varchar"} - ], - "primaryKey": "custkey" - } - ] - } - return base64.b64encode(orjson.dumps(manifest)).decode('utf-8') + manifest_str = base64.b64encode(orjson.dumps(manifest)).decode('utf-8') @staticmethod def to_connection_info(pg: PostgresContainer): @@ -81,24 +84,33 @@ def to_connection_url(pg: PostgresContainer): info = TestPostgres.to_connection_info(pg) return f"postgres://{info['user']}:{info['password']}@{info['host']}:{info['port']}/{info['database']}" - def test_query(self, postgres: PostgresContainer, manifest_str: str): + def test_query(self, postgres: PostgresContainer): connection_info = self.to_connection_info(postgres) response = client.post( url="/v2/ibis/postgres/query", json={ "connectionInfo": connection_info, - "manifestStr": manifest_str, + "manifestStr": self.manifest_str, "sql": 'SELECT * FROM "Orders" LIMIT 1' } ) assert response.status_code == 200 result = response.json() - assert len(result['columns']) == 2 + assert len(result['columns']) == len(self.manifest['models'][0]['columns']) assert len(result['data']) == 1 - assert result['data'][0][0] == 1 - assert result['dtypes'] is not None + assert result['data'][0] == [1, 370, 'O', '172799.49', 820540800000, '1_370', 1704153599000, 1704153599000] + assert result['dtypes'] == { + 'orderkey': 'int32', + 'custkey': 'int32', + 'orderstatus': 'object', + 'totalprice': 'object', + 'orderdate': 'object', + 'order_cust_key': 'object', + 'timestamp': 'datetime64[ns]', + 'timestamptz': 'datetime64[ns, UTC]' + } - def test_query_with_connection_url(self, postgres: PostgresContainer, manifest_str: str): + def test_query_with_connection_url(self, postgres: PostgresContainer): connection_url = self.to_connection_url(postgres) response = client.post( url="/v2/ibis/postgres/query", @@ -106,17 +118,49 @@ def test_query_with_connection_url(self, postgres: PostgresContainer, manifest_s "connectionInfo": { "connectionUrl": connection_url }, - "manifestStr": manifest_str, + "manifestStr": self.manifest_str, "sql": 'SELECT * FROM "Orders" LIMIT 1' } ) assert response.status_code == 200 result = response.json() - assert len(result['columns']) == 2 + assert len(result['columns']) == len(self.manifest['models'][0]['columns']) assert len(result['data']) == 1 assert result['data'][0][0] == 1 assert result['dtypes'] is not None + def test_query_with_column_dtypes(self, postgres: PostgresContainer): + connection_info = self.to_connection_info(postgres) + response = client.post( + url="/v2/ibis/postgres/query", + json={ + "connectionInfo": connection_info, + "manifestStr": self.manifest_str, + "sql": 'SELECT * FROM "Orders" LIMIT 1', + "columnDtypes": { + "totalprice": "float", + "orderdate": "datetime64", + "timestamp": "datetime64", + "timestamptz": "datetime64" + } + } + ) + assert response.status_code == 200 + result = response.json() + assert len(result['columns']) == len(self.manifest['models'][0]['columns']) + assert len(result['data']) == 1 + assert result['data'][0] == [1, 370, 'O', 172799.49, '1996-01-02 00:00:00.000000', '1_370', '2024-01-01 23:59:59.000000', '2024-01-01 23:59:59.000000 UTC'] + assert result['dtypes'] == { + 'orderkey': 'int32', + 'custkey': 'int32', + 'orderstatus': 'object', + 'totalprice': 'float64', + 'orderdate': 'object', + 'order_cust_key': 'object', + 'timestamp': 'object', + 'timestamptz': 'object' + } + def test_query_without_manifest(self, postgres: PostgresContainer): connection_info = self.to_connection_info(postgres) response = client.post( @@ -133,13 +177,13 @@ def test_query_without_manifest(self, postgres: PostgresContainer): assert result['detail'][0]['loc'] == ['body', 'manifestStr'] assert result['detail'][0]['msg'] == 'Field required' - def test_query_without_sql(self, postgres: PostgresContainer, manifest_str: str): + def test_query_without_sql(self, postgres: PostgresContainer): connection_info = self.to_connection_info(postgres) response = client.post( url="/v2/ibis/postgres/query", json={ "connectionInfo": connection_info, - "manifestStr": manifest_str + "manifestStr": self.manifest_str } ) assert response.status_code == 422 @@ -149,11 +193,11 @@ def test_query_without_sql(self, postgres: PostgresContainer, manifest_str: str) assert result['detail'][0]['loc'] == ['body', 'sql'] assert result['detail'][0]['msg'] == 'Field required' - def test_query_without_connection_info(self, manifest_str: str): + def test_query_without_connection_info(self): response = client.post( url="/v2/ibis/postgres/query", json={ - "manifestStr": manifest_str, + "manifestStr": self.manifest_str, "sql": 'SELECT * FROM "Orders" LIMIT 1' } ) @@ -164,27 +208,27 @@ def test_query_without_connection_info(self, manifest_str: str): assert result['detail'][0]['loc'] == ['body', 'connectionInfo'] assert result['detail'][0]['msg'] == 'Field required' - def test_query_with_dry_run(self, postgres: PostgresContainer, manifest_str: str): + def test_query_with_dry_run(self, postgres: PostgresContainer): connection_info = self.to_connection_info(postgres) response = client.post( url="/v2/ibis/postgres/query", params={"dryRun": True}, json={ "connectionInfo": connection_info, - "manifestStr": manifest_str, + "manifestStr": self.manifest_str, "sql": 'SELECT * FROM "Orders" LIMIT 1' } ) assert response.status_code == 204 - def test_query_with_dry_run_and_invalid_sql(self, postgres: PostgresContainer, manifest_str: str): + def test_query_with_dry_run_and_invalid_sql(self, postgres: PostgresContainer): connection_info = self.to_connection_info(postgres) response = client.post( url="/v2/ibis/postgres/query", params={"dryRun": True}, json={ "connectionInfo": connection_info, - "manifestStr": manifest_str, + "manifestStr": self.manifest_str, "sql": 'SELECT * FROM X' } ) diff --git a/ibis-server/tests/routers/ibis/test_snowflake.py b/ibis-server/tests/routers/ibis/test_snowflake.py index e15ce09cc..060a27404 100644 --- a/ibis-server/tests/routers/ibis/test_snowflake.py +++ b/ibis-server/tests/routers/ibis/test_snowflake.py @@ -1,3 +1,6 @@ +import base64 + +import orjson import pytest from fastapi.testclient import TestClient @@ -8,38 +11,39 @@ @pytest.mark.snowflake class TestSnowflake: + manifest = { + "catalog": "my_catalog", + "schema": "my_schema", + "models": [ + { + "name": "Orders", + "properties": {}, + "refSql": "select * from TPCH_SF1.ORDERS", + "columns": [ + {"name": "orderkey", "expression": "O_ORDERKEY", "type": "integer"}, + {"name": "custkey", "expression": "O_CUSTKEY", "type": "integer"}, + {"name": "orderstatus", "expression": "O_ORDERSTATUS", "type": "varchar"}, + {"name": "totalprice", "expression": "O_TOTALPRICE", "type": "float"}, + {"name": "orderdate", "expression": "O_ORDERDATE", "type": "date"}, + {"name": "order_cust_key", "expression": "concat(O_ORDERKEY, '_', O_CUSTKEY)", "type": "varchar"}, + {"name": "timestamp", "expression": "cast('2024-01-01T23:59:59' as timestamp)", "type": "timestamp"}, + {"name": "timestamptz", "expression": "cast('2024-01-01T23:59:59' as timestamp with time zone)", "type": "timestamp"} + ], + "primaryKey": "orderkey" + }, + { + "name": "Customer", + "refSql": "select * from TPCH_SF1.CUSTOMER", + "columns": [ + {"name": "custkey", "expression": "C_CUSTKEY", "type": "integer"}, + {"name": "name", "expression": "C_NAME", "type": "varchar"} + ], + "primaryKey": "custkey" + } + ] + } - @pytest.fixture() - def manifest_str(self) -> str: - import base64 - import orjson - - manifest = { - "catalog": "my_catalog", - "schema": "my_schema", - "models": [ - { - "name": "Orders", - "properties": {}, - "refSql": "select * from TPCH_SF1.ORDERS", - "columns": [ - {"name": "orderkey", "expression": "O_ORDERKEY", "type": "integer"}, - {"name": "custkey", "type": "integer", "expression": "O_CUSTKEY"} - ], - "primaryKey": "orderkey" - }, - { - "name": "Customer", - "refSql": "select * from TPCH_SF1.CUSTOMER", - "columns": [ - {"name": "custkey", "expression": "C_CUSTKEY", "type": "integer"}, - {"name": "name", "expression": "C_NAME", "type": "varchar"} - ], - "primaryKey": "custkey" - } - ] - } - return base64.b64encode(orjson.dumps(manifest)).decode('utf-8') + manifest_str = base64.b64encode(orjson.dumps(manifest)).decode('utf-8') @staticmethod def get_connection_info(): @@ -52,22 +56,63 @@ def get_connection_info(): "schema": "TPCH_SF1" } - def test_query(self, manifest_str: str): + def test_query(self): connection_info = self.get_connection_info() response = client.post( url="/v2/ibis/snowflake/query", json={ "connectionInfo": connection_info, - "manifestStr": manifest_str, - "sql": 'SELECT * FROM "Orders" LIMIT 1' + "manifestStr": self.manifest_str, + "sql": 'SELECT * FROM "Orders" ORDER BY "orderkey" LIMIT 1' + } + ) + assert response.status_code == 200 + result = response.json() + assert len(result['columns']) == len(self.manifest['models'][0]['columns']) + assert len(result['data']) == 1 + assert result['data'][0] == [1, 36901, 'O', 173665.47, 820540800000, '1_36901', 1704153599000, 1704153599000] + assert result['dtypes'] == { + 'orderkey': 'int64', + 'custkey': 'int64', + 'orderstatus': 'object', + 'totalprice': 'object', + 'orderdate': 'object', + 'order_cust_key': 'object', + 'timestamp': 'datetime64[ns]', + 'timestamptz': 'datetime64[ns, UTC]' + } + + def test_query_with_column_dtypes(self): + connection_info = self.get_connection_info() + response = client.post( + url="/v2/ibis/snowflake/query", + json={ + "connectionInfo": connection_info, + "manifestStr": self.manifest_str, + "sql": 'SELECT * FROM "Orders" ORDER BY "orderkey" LIMIT 1', + "columnDtypes": { + "totalprice": "float", + "orderdate": "datetime64", + "timestamp": "datetime64", + "timestamptz": "datetime64" + } } ) assert response.status_code == 200 result = response.json() - assert len(result['columns']) == 2 + assert len(result['columns']) == len(self.manifest['models'][0]['columns']) assert len(result['data']) == 1 - assert result['data'][0][0] is not None - assert result['dtypes'] is not None + assert result['data'][0] == [1, 36901, 'O', 173665.47, '1996-01-02 00:00:00.000000', '1_36901', '2024-01-01 23:59:59.000000', '2024-01-01 23:59:59.000000 UTC'] + assert result['dtypes'] == { + 'orderkey': 'int64', + 'custkey': 'int64', + 'orderstatus': 'object', + 'totalprice': 'float64', + 'orderdate': 'object', + 'order_cust_key': 'object', + 'timestamp': 'object', + 'timestamptz': 'object' + } def test_query_without_manifest(self): connection_info = self.get_connection_info() @@ -85,13 +130,13 @@ def test_query_without_manifest(self): assert result['detail'][0]['loc'] == ['body', 'manifestStr'] assert result['detail'][0]['msg'] == 'Field required' - def test_query_without_sql(self, manifest_str: str): + def test_query_without_sql(self): connection_info = self.get_connection_info() response = client.post( url="/v2/ibis/snowflake/query", json={ "connectionInfo": connection_info, - "manifestStr": manifest_str + "manifestStr": self.manifest_str } ) assert response.status_code == 422 @@ -101,11 +146,11 @@ def test_query_without_sql(self, manifest_str: str): assert result['detail'][0]['loc'] == ['body', 'sql'] assert result['detail'][0]['msg'] == 'Field required' - def test_query_without_connection_info(self, manifest_str: str): + def test_query_without_connection_info(self): response = client.post( url="/v2/ibis/snowflake/query", json={ - "manifestStr": manifest_str, + "manifestStr": self.manifest_str, "sql": 'SELECT * FROM "Orders" LIMIT 1' } ) @@ -116,27 +161,27 @@ def test_query_without_connection_info(self, manifest_str: str): assert result['detail'][0]['loc'] == ['body', 'connectionInfo'] assert result['detail'][0]['msg'] == 'Field required' - def test_query_with_dry_run(self, manifest_str: str): + def test_query_with_dry_run(self): connection_info = self.get_connection_info() response = client.post( url="/v2/ibis/snowflake/query", params={"dryRun": True}, json={ "connectionInfo": connection_info, - "manifestStr": manifest_str, + "manifestStr": self.manifest_str, "sql": 'SELECT * FROM "Orders" LIMIT 1' } ) assert response.status_code == 204 - def test_query_with_dry_run_and_invalid_sql(self, manifest_str: str): + def test_query_with_dry_run_and_invalid_sql(self): connection_info = self.get_connection_info() response = client.post( url="/v2/ibis/snowflake/query", params={"dryRun": True}, json={ "connectionInfo": connection_info, - "manifestStr": manifest_str, + "manifestStr": self.manifest_str, "sql": 'SELECT * FROM X' } )