fixes more tests

.github/workflows/test_common.yml

@@ -17,7 +17,7 @@ env:
 
   # we need the secrets only for the rest_api_pipeline tests which are in tests/sources
   # so we inject them only at the end
-  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
+  SOURCES__GITHUB__ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
 jobs:
   get_docs_changes:
@@ -126,9 +126,6 @@ jobs:
         name: Run pipeline tests with pyarrow but no pandas installed Windows
         shell: cmd
 
-      - name: create secrets.toml for examples
-        run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
-
       - name: Install pipeline and sources dependencies
         run: poetry install --no-interaction -E duckdb -E cli -E parquet -E deltalake -E sql_database --with sentry-sdk,pipeline,sources
 

dlt/common/destination/utils.py

@@ -15,7 +15,7 @@
     SchemaIdentifierNormalizationCollision,
 )
 from dlt.common.schema.typing import TColumnType, TLoaderMergeStrategy, TSchemaTables, TTableSchema
-from dlt.common.schema.utils import get_merge_strategy
+from dlt.common.schema.utils import get_merge_strategy, is_complete_column
 from dlt.common.storages import ParsedLoadJobFileName
 from dlt.common.typing import ConfigValue, DictStrStr, TLoaderFileFormat
 
@@ -156,6 +156,9 @@ def verify_supported_data_types(
     for table in prepared_tables:
         # map types
         for column in table["columns"].values():
+            # do not verify incomplete columns, those won't be created
+            if not is_complete_column(column):
+                continue
             try:
                 type_mapper.to_destination_type(column, table)
             except Exception as ex:

dlt/destinations/impl/dremio/factory.py

@@ -63,7 +63,7 @@ def ensure_supported_type(
             return
         if loader_file_format == "parquet":
             # binary not supported on parquet if precision is set
-            if column.get("precision") is not None:
+            if column.get("precision") is not None and column["data_type"] == "binary":
                 raise TerminalValueError(
                     "Dremio cannot load fixed width 'binary' columns from parquet files. Switch to"
                     " other file format or use binary columns without precision.",

dlt/destinations/impl/redshift/factory.py

@@ -77,7 +77,7 @@ def ensure_supported_type(
                 raise TerminalValueError("", "binary")
         if loader_file_format == "parquet":
             # binary not supported on parquet if precision is set
-            if column.get("precision"):
+            if column.get("precision") and column["data_type"] == "binary":
                 raise TerminalValueError(
                     "Redshift cannot load fixed width VARBYTE columns from parquet files. Switch"
                     " to other file format or use binary columns without precision.",

pyproject.toml

@@ -165,8 +165,8 @@ mimesis = "^7.0.0"
 optional = true
 [tool.poetry.group.sources.dependencies]
 connectorx = [
-    {version = "0.3.2", python = "^3.8", optional = true},
-    {version = ">=0.3.2", python = ">=3.9", optional = true}
+    {version = "0.3.2", python = "^3.8"},
+    {version = ">=0.3.2", python = ">=3.9"}
 ]
 pymysql = "^1.1.0"
 openpyxl = "^3"

tests/libs/test_parquet_writer.py

@@ -296,7 +296,7 @@ def _assert_arrow_field(field: int, prec: str) -> None:
             else:
                 assert column_type.tz is None
 
-        _assert_arrow_field(0, "us")
+        _assert_arrow_field(0, "s")
         _assert_arrow_field(1, "ms")
         _assert_arrow_field(2, "us")
         _assert_arrow_field(3, "ns")
@@ -306,10 +306,12 @@ def _assert_arrow_field(field: int, prec: str) -> None:
 
         def _assert_pq_column(col: int, prec: str) -> None:
             info = json.loads(reader.metadata.schema.column(col).logical_type.to_json())
+            print(info)
             assert info["isAdjustedToUTC"] is adjusted
             assert info["timeUnit"] == prec
 
-        _assert_pq_column(0, "microseconds")
+        # apparently storting seconds is not supported
+        _assert_pq_column(0, "milliseconds")
         _assert_pq_column(1, "milliseconds")
         _assert_pq_column(2, "microseconds")
         _assert_pq_column(3, "nanoseconds")

tests/load/pipeline/test_pipelines.py

@@ -512,7 +512,7 @@ def other_data():
 
     # duckdb 0.9.1 does not support TIME other than 6
     if destination_config.destination in ["duckdb", "motherduck"]:
-        column_schemas["col11_precision"]["precision"] = 0
+        column_schemas["col11_precision"]["precision"] = None
         # also we do not want to test col4_precision (datetime) because
         # those timestamps are not TZ aware in duckdb and we'd need to
         # disable TZ when generating parquet

tests/load/sources/sql_database/test_sql_database_source.py

@@ -969,11 +969,12 @@ def assert_row_counts(
     tables: Optional[List[str]] = None,
     include_views: bool = False,
 ) -> None:
-    tables = [
-        tbl_name
-        for tbl_name, info in sql_source_db.table_infos.items()
-        if include_views or not info["is_view"]
-    ]
+    if not tables:
+        tables = [
+            tbl_name
+            for tbl_name, info in sql_source_db.table_infos.items()
+            if include_views or not info["is_view"]
+        ]
     dest_counts = load_table_counts(pipeline, *tables)
     for table in tables:
         info = sql_source_db.table_infos[table]

tests/sources/rest_api/test_rest_api_pipeline_template.py

@@ -1,9 +1,9 @@
+import os
 import dlt
 import pytest
 from dlt.common.typing import TSecretStrValue
 
 
-# NOTE: needs github secrets to work
 @pytest.mark.parametrize(
     "example_name",
     (
@@ -16,5 +16,8 @@ def test_all_examples(example_name: str) -> None:
 
     # reroute token location from secrets
     github_token: TSecretStrValue = dlt.secrets.get("sources.github.access_token")
+    if not github_token:
+        # try to get GITHUB TOKEN which is available on github actions, fallback to None if not available
+        github_token = os.environ.get("GITHUB_TOKEN", None)  # type: ignore
     dlt.secrets["sources.rest_api_pipeline.github.access_token"] = github_token
     getattr(rest_api_pipeline, example_name)()