diff --git a/dlt/common/libs/sql_alchemy.py b/dlt/common/libs/sql_alchemy.py index c3cc85f5c2..f96b57b415 100644 --- a/dlt/common/libs/sql_alchemy.py +++ b/dlt/common/libs/sql_alchemy.py @@ -9,9 +9,13 @@ from sqlalchemy.sql import sqltypes, Select from sqlalchemy.sql.sqltypes import TypeEngine from sqlalchemy.exc import CompileError + import sqlalchemy as sa except ModuleNotFoundError: raise MissingDependencyException( "dlt sql_database helpers ", [f"{version.DLT_PKG_NAME}[sql_database]"], "Install the sql_database helpers for loading from sql_database sources.", ) + +# TODO: maybe use sa.__version__? +IS_SQL_ALCHEMY_20 = hasattr(sa, "Double") diff --git a/tests/load/sources/sql_database/test_sql_database_source.py b/tests/load/sources/sql_database/test_sql_database_source.py index 423a8a54b5..d6c769b486 100644 --- a/tests/load/sources/sql_database/test_sql_database_source.py +++ b/tests/load/sources/sql_database/test_sql_database_source.py @@ -407,7 +407,7 @@ def test_type_adapter_callback( def conversion_callback(t): if isinstance(t, sa.JSON): return sa.Text - elif hasattr(sa, "Double") and isinstance(t, sa.Double): # type: ignore[attr-defined] + elif hasattr(sa, "Double") and isinstance(t, sa.Double): return sa.BIGINT return t diff --git a/tests/load/sources/sql_database/test_sql_database_source_all_destinations.py b/tests/load/sources/sql_database/test_sql_database_source_all_destinations.py index 4acad09bcc..7012602b4a 100644 --- a/tests/load/sources/sql_database/test_sql_database_source_all_destinations.py +++ b/tests/load/sources/sql_database/test_sql_database_source_all_destinations.py @@ -27,6 +27,7 @@ default_test_callback, ) from tests.load.sources.sql_database.sql_source import SQLAlchemySourceDB + from dlt.common.libs.sql_alchemy import IS_SQL_ALCHEMY_20 except MissingDependencyException: pytest.skip("Tests require sql alchemy", allow_module_level=True) @@ -155,6 +156,9 @@ def test_load_sql_table_incremental( """ os.environ["SOURCES__SQL_DATABASE__CHAT_MESSAGE__INCREMENTAL__CURSOR_PATH"] = "updated_at" + if not IS_SQL_ALCHEMY_20 and backend == "connectorx": + pytest.skip("Test will not run on sqlalchemy 1.4 with connectorx") + pipeline = destination_config.setup_pipeline(request.node.name, dev_mode=True) tables = ["chat_message"] @@ -280,6 +284,9 @@ def test_load_sql_table_resource_incremental( backend: TableBackend, request: Any, ) -> None: + if not IS_SQL_ALCHEMY_20 and backend == "connectorx": + pytest.skip("Test will not run on sqlalchemy 1.4 with connectorx") + @dlt.source def sql_table_source() -> List[DltResource]: return [ @@ -315,6 +322,9 @@ def test_load_sql_table_resource_incremental_initial_value( backend: TableBackend, request: Any, ) -> None: + if not IS_SQL_ALCHEMY_20 and backend == "connectorx": + pytest.skip("Test will not run on sqlalchemy 1.4 with connectorx") + @dlt.source def sql_table_source() -> List[DltResource]: return [