fix delta table format destination config selection

dlt-hub · Aug 1, 2024 · 65737e7 · 65737e7
1 parent b07dddc
commit 65737e7
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 17 deletions.
diff --git a/tests/load/pipeline/test_filesystem_pipeline.py b/tests/load/pipeline/test_filesystem_pipeline.py
@@ -249,7 +249,7 @@ def foo():
 @pytest.mark.parametrize(
     "destination_config",
     destinations_configs(
-        all_buckets_filesystem_configs=True,
+        table_format_filesystem_configs=True,
         table_format="delta",
         bucket_exclude=(MEMORY_BUCKET),
     ),
@@ -312,8 +312,9 @@ def data_types():
 @pytest.mark.parametrize(
     "destination_config",
     destinations_configs(
-        local_filesystem_configs=True,
+        table_format_filesystem_configs=True,
         table_format="delta",
+        bucket_subset=(FILE_BUCKET),
     ),
     ids=lambda x: x.name,
 )
@@ -358,8 +359,9 @@ def delta_table():
 @pytest.mark.parametrize(
     "destination_config",
     destinations_configs(
-        local_filesystem_configs=True,
+        table_format_filesystem_configs=True,
         table_format="delta",
+        bucket_subset=(FILE_BUCKET),
     ),
     ids=lambda x: x.name,
 )
@@ -434,8 +436,9 @@ def complex_table():
 @pytest.mark.parametrize(
     "destination_config",
     destinations_configs(
-        local_filesystem_configs=True,
+        table_format_filesystem_configs=True,
         table_format="delta",
+        bucket_subset=(FILE_BUCKET),
     ),
     ids=lambda x: x.name,
 )
@@ -524,8 +527,9 @@ def delta_table(data):
 @pytest.mark.parametrize(
     "destination_config",
     destinations_configs(
-        local_filesystem_configs=True,
+        table_format_filesystem_configs=True,
         table_format="delta",
+        bucket_subset=(FILE_BUCKET),
     ),
     ids=lambda x: x.name,
 )
@@ -572,8 +576,9 @@ def s():
 @pytest.mark.parametrize(
     "destination_config",
     destinations_configs(
-        local_filesystem_configs=True,
+        table_format_filesystem_configs=True,
         table_format="delta",
+        bucket_subset=(FILE_BUCKET),
     ),
     ids=lambda x: x.name,
 )

diff --git a/tests/load/pipeline/test_merge_disposition.py b/tests/load/pipeline/test_merge_disposition.py
@@ -58,6 +58,7 @@ def skip_if_not_supported(
     destinations_configs(
         default_sql_configs=True,
         all_buckets_filesystem_configs=True,
+        table_format_filesystem_configs=True,
         supports_merge=True,
         bucket_subset=(FILE_BUCKET, AZ_BUCKET),  # test one local, one remote
     ),
@@ -138,7 +139,9 @@ def data(slice_: slice = None):
     destinations_configs(
         default_sql_configs=True,
         local_filesystem_configs=True,
+        table_format_filesystem_configs=True,
         supports_merge=True,
+        bucket_subset=(FILE_BUCKET),
     ),
     ids=lambda x: x.name,
 )
@@ -248,7 +251,11 @@ def r(data):
 @pytest.mark.parametrize(
     "destination_config",
     destinations_configs(
-        default_sql_configs=True, local_filesystem_configs=True, supports_merge=True
+        default_sql_configs=True,
+        local_filesystem_configs=True,
+        table_format_filesystem_configs=True,
+        supports_merge=True,
+        bucket_subset=(FILE_BUCKET),
     ),
     ids=lambda x: x.name,
 )
@@ -732,7 +739,11 @@ def duplicates_no_child():
 @pytest.mark.parametrize(
     "destination_config",
     destinations_configs(
-        default_sql_configs=True, local_filesystem_configs=True, supports_merge=True
+        default_sql_configs=True,
+        local_filesystem_configs=True,
+        table_format_filesystem_configs=True,
+        supports_merge=True,
+        bucket_subset=(FILE_BUCKET),
     ),
     ids=lambda x: x.name,
 )
@@ -1207,7 +1218,7 @@ def r():
     "destination_config",
     destinations_configs(
         default_sql_configs=True,
-        local_filesystem_configs=True,
+        table_format_filesystem_configs=True,
         supports_merge=True,
         subset=["postgres", "filesystem"],  # test one SQL and one non-SQL destination
     ),

diff --git a/tests/load/utils.py b/tests/load/utils.py
@@ -211,6 +211,7 @@ def destinations_configs(
     all_staging_configs: bool = False,
     local_filesystem_configs: bool = False,
     all_buckets_filesystem_configs: bool = False,
+    table_format_filesystem_configs: bool = False,
     subset: Sequence[str] = (),
     bucket_subset: Sequence[str] = (),
     exclude: Sequence[str] = (),
@@ -494,14 +495,6 @@ def destinations_configs(
                 supports_merge=False,
             )
         ]
-        destination_configs += [
-            DestinationTestConfiguration(
-                destination="filesystem",
-                bucket_url=FILE_BUCKET,
-                table_format="delta",
-                supports_merge=True,
-            )
-        ]
 
     if all_buckets_filesystem_configs:
         for bucket in DEFAULT_BUCKETS:
@@ -513,6 +506,9 @@ def destinations_configs(
                     supports_merge=False,
                 )
             ]
+
+    if table_format_filesystem_configs:
+        for bucket in DEFAULT_BUCKETS:
             destination_configs += [
                 DestinationTestConfiguration(
                     destination="filesystem",