re-enable delta tests for read access

add rudimentary tests for destination configs
fixes small problems in test setup

tests/load/utils.py

@@ -374,6 +374,7 @@ def destinations_configs(
                 file_format="parquet",
                 bucket_url=AWS_BUCKET,
                 supports_dbt=False,
+                extra_info="minio",
             )
         ]
         destination_configs += [
@@ -439,7 +440,7 @@ def destinations_configs(
                 file_format="jsonl",
                 bucket_url=AWS_BUCKET,
                 stage_name="PUBLIC.dlt_s3_stage",
-                extra_info="s3-integration",
+                extra_info="s3-integration-public-stage",
             ),
             DestinationTestConfiguration(
                 destination_type="snowflake",
@@ -515,13 +516,6 @@ def destinations_configs(
                 bucket_url=AWS_BUCKET,
                 extra_info="s3-authorization",
             ),
-            DestinationTestConfiguration(
-                destination_type="dremio",
-                staging=filesystem(destination_name="minio"),
-                file_format="parquet",
-                bucket_url=AWS_BUCKET,
-                supports_dbt=False,
-            ),
         ]
 
     if all_staging_configs:
@@ -612,6 +606,7 @@ def destinations_configs(
                     extra_info=bucket + "-delta",
                     table_format="delta",
                     supports_merge=True,
+                    file_format="parquet",
                     env_vars=(
                         {
                             "DESTINATION__FILESYSTEM__DELTALAKE_STORAGE_OPTIONS": (

tests/tests/__init__.py

@@ -0,0 +1 @@
+"""Tests for our test helpers"""

tests/tests/load/test_utils.py

@@ -0,0 +1,49 @@
+# NOTE: these should be run with all destinations enabled
+# NOTE: these are very rudimentary tests, they should be extended
+
+from typing import Any
+
+from tests.load.utils import destinations_configs, DEFAULT_BUCKETS, DestinationTestConfiguration
+
+
+def test_empty_destinations_configs():
+    configs = destinations_configs()
+    assert len(configs) == 0
+
+
+def _assert_name_uniqueness(configs: Any) -> None:
+    identifiers = [config[0][0].name for config in configs]
+    print(identifiers)
+    assert len(identifiers) == len(set(identifiers)), "Identifier uniqueness violated"
+
+
+def test_enable_filesystem_configs():
+    # enable local filesystem configs
+    configs = destinations_configs(local_filesystem_configs=True)
+    _assert_name_uniqueness(configs)
+    assert len(configs) == 3
+
+    # enable all buckets configs
+    configs = destinations_configs(all_buckets_filesystem_configs=True)
+    _assert_name_uniqueness(configs)
+    assert len(configs) == len(DEFAULT_BUCKETS) == 7  # hardcoded for now
+
+    # enable with delta tables
+    configs = destinations_configs(
+        all_buckets_filesystem_configs=True, table_format_filesystem_configs=True
+    )
+    _assert_name_uniqueness(configs)
+    assert (
+        len(configs) == len(DEFAULT_BUCKETS) * 2 == 7 * 2
+    )  # we have delta now, so double the expected amount
+
+
+def test_uniqueness_of_names():
+    configs = destinations_configs(
+        default_sql_configs=True,
+        default_vector_configs=True,
+        default_staging_configs=True,
+        all_staging_configs=True,
+        all_buckets_filesystem_configs=True,
+    )
+    _assert_name_uniqueness(configs)