diff --git a/tests/load/pipeline/test_filesystem_pipeline.py b/tests/load/pipeline/test_filesystem_pipeline.py index 308a1d516c..7ad571f2aa 100644 --- a/tests/load/pipeline/test_filesystem_pipeline.py +++ b/tests/load/pipeline/test_filesystem_pipeline.py @@ -249,7 +249,7 @@ def foo(): @pytest.mark.parametrize( "destination_config", destinations_configs( - all_buckets_filesystem_configs=True, + table_format_filesystem_configs=True, table_format="delta", bucket_exclude=(MEMORY_BUCKET), ), @@ -312,8 +312,9 @@ def data_types(): @pytest.mark.parametrize( "destination_config", destinations_configs( - local_filesystem_configs=True, + table_format_filesystem_configs=True, table_format="delta", + bucket_subset=(FILE_BUCKET), ), ids=lambda x: x.name, ) @@ -358,8 +359,9 @@ def delta_table(): @pytest.mark.parametrize( "destination_config", destinations_configs( - local_filesystem_configs=True, + table_format_filesystem_configs=True, table_format="delta", + bucket_subset=(FILE_BUCKET), ), ids=lambda x: x.name, ) @@ -434,8 +436,9 @@ def complex_table(): @pytest.mark.parametrize( "destination_config", destinations_configs( - local_filesystem_configs=True, + table_format_filesystem_configs=True, table_format="delta", + bucket_subset=(FILE_BUCKET), ), ids=lambda x: x.name, ) @@ -524,8 +527,9 @@ def delta_table(data): @pytest.mark.parametrize( "destination_config", destinations_configs( - local_filesystem_configs=True, + table_format_filesystem_configs=True, table_format="delta", + bucket_subset=(FILE_BUCKET), ), ids=lambda x: x.name, ) @@ -572,8 +576,9 @@ def s(): @pytest.mark.parametrize( "destination_config", destinations_configs( - local_filesystem_configs=True, + table_format_filesystem_configs=True, table_format="delta", + bucket_subset=(FILE_BUCKET), ), ids=lambda x: x.name, ) diff --git a/tests/load/pipeline/test_merge_disposition.py b/tests/load/pipeline/test_merge_disposition.py index 68f14b53d1..63188d4f5e 100644 --- a/tests/load/pipeline/test_merge_disposition.py +++ b/tests/load/pipeline/test_merge_disposition.py @@ -58,6 +58,7 @@ def skip_if_not_supported( destinations_configs( default_sql_configs=True, all_buckets_filesystem_configs=True, + table_format_filesystem_configs=True, supports_merge=True, bucket_subset=(FILE_BUCKET, AZ_BUCKET), # test one local, one remote ), @@ -138,7 +139,9 @@ def data(slice_: slice = None): destinations_configs( default_sql_configs=True, local_filesystem_configs=True, + table_format_filesystem_configs=True, supports_merge=True, + bucket_subset=(FILE_BUCKET), ), ids=lambda x: x.name, ) @@ -248,7 +251,11 @@ def r(data): @pytest.mark.parametrize( "destination_config", destinations_configs( - default_sql_configs=True, local_filesystem_configs=True, supports_merge=True + default_sql_configs=True, + local_filesystem_configs=True, + table_format_filesystem_configs=True, + supports_merge=True, + bucket_subset=(FILE_BUCKET), ), ids=lambda x: x.name, ) @@ -732,7 +739,11 @@ def duplicates_no_child(): @pytest.mark.parametrize( "destination_config", destinations_configs( - default_sql_configs=True, local_filesystem_configs=True, supports_merge=True + default_sql_configs=True, + local_filesystem_configs=True, + table_format_filesystem_configs=True, + supports_merge=True, + bucket_subset=(FILE_BUCKET), ), ids=lambda x: x.name, ) @@ -1207,7 +1218,7 @@ def r(): "destination_config", destinations_configs( default_sql_configs=True, - local_filesystem_configs=True, + table_format_filesystem_configs=True, supports_merge=True, subset=["postgres", "filesystem"], # test one SQL and one non-SQL destination ), diff --git a/tests/load/utils.py b/tests/load/utils.py index 66211b1701..4b6c01c916 100644 --- a/tests/load/utils.py +++ b/tests/load/utils.py @@ -211,6 +211,7 @@ def destinations_configs( all_staging_configs: bool = False, local_filesystem_configs: bool = False, all_buckets_filesystem_configs: bool = False, + table_format_filesystem_configs: bool = False, subset: Sequence[str] = (), bucket_subset: Sequence[str] = (), exclude: Sequence[str] = (), @@ -494,14 +495,6 @@ def destinations_configs( supports_merge=False, ) ] - destination_configs += [ - DestinationTestConfiguration( - destination="filesystem", - bucket_url=FILE_BUCKET, - table_format="delta", - supports_merge=True, - ) - ] if all_buckets_filesystem_configs: for bucket in DEFAULT_BUCKETS: @@ -513,6 +506,9 @@ def destinations_configs( supports_merge=False, ) ] + + if table_format_filesystem_configs: + for bucket in DEFAULT_BUCKETS: destination_configs += [ DestinationTestConfiguration( destination="filesystem",