master merge for 0.4.6 release

.github/workflows/test_common.yml

@@ -83,11 +83,11 @@ jobs:
         run: poetry install --no-interaction -E duckdb --with sentry-sdk
 
       - run: |
-          poetry run pytest tests/pipeline/test_pipeline.py
+          poetry run pytest tests/pipeline/test_pipeline.py tests/pipeline/test_import_export_schema.py
         if: runner.os != 'Windows'
         name: Run pipeline smoke tests with minimum deps Linux/MAC
       - run: |
-          poetry run pytest tests/pipeline/test_pipeline.py
+          poetry run pytest tests/pipeline/test_pipeline.py tests/pipeline/test_import_export_schema.py
         if: runner.os == 'Windows'
         name: Run smoke tests with minimum deps Windows
         shell: cmd

.github/workflows/test_destination_athena_iceberg.yml

@@ -65,7 +65,7 @@ jobs:
 
       - name: Install dependencies
         # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install --no-interaction -E --with sentry-sdk --with pipeline
+        run: poetry install --no-interaction -E athena --with sentry-sdk --with pipeline
 
       - name: create secrets.toml
         run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml

.github/workflows/test_doc_snippets.yml

@@ -17,7 +17,8 @@ env:
 
   # Slack hook for chess in production example
   RUNTIME__SLACK_INCOMING_HOOK: ${{ secrets.RUNTIME__SLACK_INCOMING_HOOK }}
-
+  # detect if the workflow is executed in a repo fork
+  IS_FORK: ${{ github.event.pull_request.head.repo.fork }}
 jobs:
 
   run_lint:

Makefile

@@ -24,7 +24,7 @@ help:
 	@echo "		test"
 	@echo "			tests all the components including destinations"
 	@echo "		test-load-local"
-	@echo "			tests all components unsing local destinations: duckdb and postgres"
+	@echo "			tests all components using local destinations: duckdb and postgres"
 	@echo "		test-common"
 	@echo "			tests common components"
 	@echo "		test-and-lint-snippets"

dlt/__main__.py

@@ -0,0 +1,4 @@
+from dlt.cli._dlt import main
+
+if __name__ == "__main__":
+    main()

dlt/common/configuration/specs/connection_string_credentials.py

@@ -1,5 +1,5 @@
 from typing import Any, ClassVar, Dict, List, Optional
-from sqlalchemy.engine import URL, make_url
+from dlt.common.libs.sql_alchemy import URL, make_url
 from dlt.common.configuration.specs.exceptions import InvalidConnectionString
 
 from dlt.common.typing import TSecretValue
@@ -26,6 +26,7 @@ def parse_native_representation(self, native_value: Any) -> None:
             # update only values that are not None
             self.update({k: v for k, v in url._asdict().items() if v is not None})
             if self.query is not None:
+                # query may be immutable so make it mutable
                 self.query = dict(self.query)
         except Exception:
             raise InvalidConnectionString(self.__class__, native_value, self.drivername)

dlt/common/data_types/type_helpers.py

@@ -13,7 +13,6 @@
 from dlt.common.data_types.typing import TDataType
 from dlt.common.time import (
     ensure_pendulum_datetime,
-    parse_iso_like_datetime,
     ensure_pendulum_date,
     ensure_pendulum_time,
 )

dlt/common/libs/numpy.py

@@ -0,0 +1,6 @@
+from dlt.common.exceptions import MissingDependencyException
+
+try:
+    import numpy
+except ModuleNotFoundError:
+    raise MissingDependencyException("DLT Numpy Helpers", ["numpy"])

dlt/common/libs/pyarrow.py

@@ -1,4 +1,5 @@
 from datetime import datetime, date  # noqa: I251
+from pendulum.tz import UTC
 from typing import Any, Tuple, Optional, Union, Callable, Iterable, Iterator, Sequence, Tuple
 
 from dlt import version
@@ -14,6 +15,7 @@
 try:
     import pyarrow
     import pyarrow.parquet
+    import pyarrow.compute
 except ModuleNotFoundError:
     raise MissingDependencyException(
         "dlt parquet Helpers", [f"{version.DLT_PKG_NAME}[parquet]"], "dlt Helpers for for parquet."
@@ -314,21 +316,21 @@ def is_arrow_item(item: Any) -> bool:
     return isinstance(item, (pyarrow.Table, pyarrow.RecordBatch))
 
 
-def to_arrow_compute_input(value: Any, arrow_type: pyarrow.DataType) -> Any:
+def to_arrow_scalar(value: Any, arrow_type: pyarrow.DataType) -> Any:
     """Converts python value to an arrow compute friendly version"""
     return pyarrow.scalar(value, type=arrow_type)
 
 
-def from_arrow_compute_output(arrow_value: pyarrow.Scalar) -> Any:
-    """Converts arrow scalar into Python type. Currently adds "UTC" to naive date times."""
+def from_arrow_scalar(arrow_value: pyarrow.Scalar) -> Any:
+    """Converts arrow scalar into Python type. Currently adds "UTC" to naive date times and converts all others to UTC"""
     row_value = arrow_value.as_py()
     # dates are not represented as datetimes but I see connector-x represents
     # datetimes as dates and keeping the exact time inside. probably a bug
     # but can be corrected this way
     if isinstance(row_value, date) and not isinstance(row_value, datetime):
         row_value = pendulum.from_timestamp(arrow_value.cast(pyarrow.int64()).as_py() / 1000)
     elif isinstance(row_value, datetime):
-        row_value = pendulum.instance(row_value)
+        row_value = pendulum.instance(row_value).in_tz("UTC")
     return row_value