add dremio native implementation for full frames and tables

dlt/destinations/impl/databricks/sql_client.py

@@ -1,5 +1,17 @@
 from contextlib import contextmanager, suppress
-from typing import Any, AnyStr, ClassVar, Generator, Iterator, Optional, Sequence, List, Tuple, Union, Dict
+from typing import (
+    Any,
+    AnyStr,
+    ClassVar,
+    Generator,
+    Iterator,
+    Optional,
+    Sequence,
+    List,
+    Tuple,
+    Union,
+    Dict,
+)
 
 
 from databricks import sql as databricks_lib
@@ -44,7 +56,8 @@ def iter_arrow(self, chunk_size: int) -> Generator[ArrowTable, None, None]:
     def iter_df(self, chunk_size: int) -> Generator[DataFrame, None, None]:
         for table in self.iter_arrow(chunk_size=chunk_size):
             yield table.to_pandas()
-
+
+
 class DatabricksSqlClient(SqlClientBase[DatabricksSqlConnection], DBTransaction):
     dbapi: ClassVar[DBApi] = databricks_lib
 

dlt/destinations/impl/dremio/sql_client.py

@@ -26,9 +26,14 @@ class DremioCursorImpl(DBApiCursorImpl):
 
     def df(self, chunk_size: int = None, **kwargs: Any) -> Optional[DataFrame]:
         if chunk_size is None:
-            return self.native_cursor.fetch_arrow_table().to_pandas()
+            return self.arrow(chunk_size=chunk_size).to_pandas()
         return super().df(chunk_size=chunk_size, **kwargs)
 
+    def arrow(self, chunk_size: int = None, **kwargs: Any) -> Optional[DataFrame]:
+        if chunk_size is None:
+            return self.native_cursor.fetch_arrow_table()
+        return super().arrow(chunk_size=chunk_size, **kwargs)
+
 
 class DremioSqlClient(SqlClientBase[pydremio.DremioConnection]):
     dbapi: ClassVar[DBApi] = pydremio