Adjust tests

dlt-hub · Apr 3, 2024 · 2613ebd · 2613ebd
1 parent e153a61
commit 2613ebd
Show file tree

Hide file tree

Showing 2 changed files with 45 additions and 13 deletions.
diff --git a/tests/load/filesystem/test_filesystem_client.py b/tests/load/filesystem/test_filesystem_client.py
@@ -11,6 +11,7 @@
     FilesystemDestinationClientConfiguration,
 )
 
+from dlt.destinations.path_utils import create_path
 from tests.load.filesystem.utils import perform_load
 from tests.utils import clean_test_storage, init_test_logging
 from tests.utils import preserve_environ, autouse_test_storage
@@ -93,6 +94,7 @@ def test_replace_write_disposition(layout: str, default_buckets_env: str) -> Non
         os.environ["DESTINATION__FILESYSTEM__LAYOUT"] = layout
     else:
         os.environ.pop("DESTINATION__FILESYSTEM__LAYOUT", None)
+
     dataset_name = "test_" + uniq_id()
     # NOTE: context manager will delete the dataset at the end so keep it open until the end
     with perform_load(dataset_name, NORMALIZED_FILES, write_disposition="replace") as load_info:
@@ -102,8 +104,14 @@ def test_replace_write_disposition(layout: str, default_buckets_env: str) -> Non
         # this path will be kept after replace
         job_2_load_1_path = posixpath.join(
             root_path,
-            LoadFilesystemJob.make_destination_filename(
-                layout, NORMALIZED_FILES[1], client.schema.name, load_id1
+            create_path(
+                layout,
+                NORMALIZED_FILES[1],
+                client.schema.name,
+                load_id1,
+                current_datetime=client.config.current_datetime,
+                datetime_format=client.config.datetime_format,
+                extra_placeholders=client.config.extra_placeholders,
             ),
         )
 
@@ -115,8 +123,14 @@ def test_replace_write_disposition(layout: str, default_buckets_env: str) -> Non
             # this one we expect to be replaced with
             job_1_load_2_path = posixpath.join(
                 root_path,
-                LoadFilesystemJob.make_destination_filename(
-                    layout, NORMALIZED_FILES[0], client.schema.name, load_id2
+                create_path(
+                    layout,
+                    NORMALIZED_FILES[0],
+                    client.schema.name,
+                    load_id2,
+                    current_datetime=client.config.current_datetime,
+                    datetime_format=client.config.datetime_format,
+                    extra_placeholders=client.config.extra_placeholders,
                 ),
             )
 
@@ -147,13 +161,24 @@ def test_append_write_disposition(layout: str, default_buckets_env: str) -> None
             client, jobs2, root_path, load_id2 = load_info
             layout = client.config.layout
             expected_files = [
-                LoadFilesystemJob.make_destination_filename(
-                    layout, job.file_name(), client.schema.name, load_id1
+                create_path(
+                    layout,
+                    job.file_name(),
+                    client.schema.name,
+                    current_datetime=client.config.current_datetime,
+                    datetime_format=client.config.datetime_format,
+                    extra_placeholders=client.config.extra_placeholders,
                 )
                 for job in jobs1
             ] + [
-                LoadFilesystemJob.make_destination_filename(
-                    layout, job.file_name(), client.schema.name, load_id2
+                create_path(
+                    layout,
+                    job.file_name(),
+                    client.schema.name,
+                    load_id2,
+                    current_datetime=client.config.current_datetime,
+                    datetime_format=client.config.datetime_format,
+                    extra_placeholders=client.config.extra_placeholders,
                 )
                 for job in jobs2
             ]

diff --git a/tests/load/pipeline/test_filesystem_pipeline.py b/tests/load/pipeline/test_filesystem_pipeline.py
@@ -1,12 +1,14 @@
+import os
 import posixpath
 from pathlib import Path
 
-import dlt, os
+import dlt
 from dlt.common.utils import uniq_id
 from dlt.common.storages.load_storage import LoadJobInfo
-from dlt.destinations.impl.filesystem.filesystem import FilesystemClient, LoadFilesystemJob
+from dlt.destinations.impl.filesystem.filesystem import FilesystemClient
 from dlt.common.schema.typing import LOADS_TABLE_NAME
 
+from dlt.destinations.path_utils import create_path
 from tests.utils import skip_if_not_active
 
 skip_if_not_active("filesystem")
@@ -18,9 +20,14 @@ def assert_file_matches(
     """Verify file contents of load job are identical to the corresponding file in destination"""
     local_path = Path(job.file_path)
     filename = local_path.name
-
-    destination_fn = LoadFilesystemJob.make_destination_filename(
-        layout, filename, client.schema.name, load_id
+    destination_fn = create_path(
+        layout,
+        filename,
+        client.schema.name,
+        load_id,
+        current_datetime=client.config.current_datetime,
+        datetime_format=client.config.datetime_format,
+        extra_placeholders=client.config.extra_placeholders,
     )
     destination_path = posixpath.join(client.dataset_path, destination_fn)