diff --git a/dlt/destinations/dataset.py b/dlt/destinations/dataset.py index 50224f65d0..f2922d8e36 100644 --- a/dlt/destinations/dataset.py +++ b/dlt/destinations/dataset.py @@ -25,13 +25,15 @@ def __init__( self.client = client self.query = query self.table = table - self.columns = columns + self.schema_columns = columns @contextmanager def cursor(self) -> Generator[SupportsReadableRelation, Any, Any]: """Gets a DBApiCursor for the current relation""" if self.table: - with self.client.table_relation(table=self.table, columns=self.columns) as cursor: + with self.client.table_relation( + table=self.table, columns=self.schema_columns + ) as cursor: yield cursor elif self.query: with self.client.query_relation(query=self.query) as cursor: diff --git a/dlt/destinations/impl/filesystem/filesystem.py b/dlt/destinations/impl/filesystem/filesystem.py index c4fbbaf71a..67bb64bb84 100644 --- a/dlt/destinations/impl/filesystem/filesystem.py +++ b/dlt/destinations/impl/filesystem/filesystem.py @@ -709,7 +709,7 @@ def table_relation( ) -> Generator[DBApiCursor, Any, Any]: table = self.sql_client.make_qualified_table_name(table) with self.sql_client.execute_query(f"SELECT * FROM {table}") as cursor: - cursor.columns = columns + cursor.schema_columns = columns yield cursor @contextmanager diff --git a/dlt/destinations/impl/mssql/mssql.py b/dlt/destinations/impl/mssql/mssql.py index 4ba7b6e8ca..dcac1e940e 100644 --- a/dlt/destinations/impl/mssql/mssql.py +++ b/dlt/destinations/impl/mssql/mssql.py @@ -151,7 +151,7 @@ def table_relation( query = f"SELECT * FROM {table}" sql_client._conn.autocommit = False with sql_client.execute_query(query) as cursor: - cursor.columns = columns + cursor.schema_columns = columns yield cursor @contextmanager diff --git a/dlt/destinations/impl/sqlalchemy/db_api_client.py b/dlt/destinations/impl/sqlalchemy/db_api_client.py index 580a7df007..7a25a6b557 100644 --- a/dlt/destinations/impl/sqlalchemy/db_api_client.py +++ b/dlt/destinations/impl/sqlalchemy/db_api_client.py @@ -71,6 +71,8 @@ def _wrap(self: "SqlalchemyClient", *args: Any, **kwargs: Any) -> Any: class SqlaDbApiCursor(DBApiCursorImpl): def __init__(self, curr: sa.engine.CursorResult) -> None: + self.schema_columns = None + # Sqlalchemy CursorResult is *mostly* compatible with DB-API cursor self.native_cursor = curr # type: ignore[assignment] curr.columns diff --git a/dlt/destinations/job_client_impl.py b/dlt/destinations/job_client_impl.py index c626b613b8..af190baae1 100644 --- a/dlt/destinations/job_client_impl.py +++ b/dlt/destinations/job_client_impl.py @@ -721,7 +721,7 @@ def table_relation( table = sql_client.make_qualified_table_name(table) query = f"SELECT * FROM {table}" with sql_client.execute_query(query) as cursor: - cursor.columns = columns + cursor.schema_columns = columns yield cursor @contextmanager diff --git a/dlt/destinations/sql_client.py b/dlt/destinations/sql_client.py index a75be97340..62e8e5e56d 100644 --- a/dlt/destinations/sql_client.py +++ b/dlt/destinations/sql_client.py @@ -322,7 +322,7 @@ class DBApiCursorImpl(DBApiCursor): def __init__(self, curr: DBApiCursor) -> None: self.native_cursor = curr - self.columns = None + self.schema_columns = None # wire protocol methods self.execute = curr.execute # type: ignore @@ -381,7 +381,7 @@ def iter_arrow(self, chunk_size: int) -> Generator[ArrowTable, None, None]: caps = DestinationCapabilitiesContext.generic_capabilities() # provide default columns in case not known - columns = self.columns or cast( + columns = self.schema_columns or cast( TTableSchemaColumns, {c: {"name": c, "nullable": True} for c in self._get_columns()} ) diff --git a/tests/load/utils.py b/tests/load/utils.py index c6718792a2..1d5a99516f 100644 --- a/tests/load/utils.py +++ b/tests/load/utils.py @@ -343,7 +343,7 @@ def destinations_configs( supports_merge=False, supports_dbt=False, destination_name="sqlalchemy_sqlite", - credentials="sqlite:///_storage/dl_data.sqlite" + credentials="sqlite:///_storage/dl_data.sqlite", ), ]