From 2613ebd83ce173dddcb737d9101f747cfc8f33b4 Mon Sep 17 00:00:00 2001 From: Sultan Iman Date: Wed, 3 Apr 2024 17:34:40 +0200 Subject: [PATCH] Adjust tests --- .../load/filesystem/test_filesystem_client.py | 41 +++++++++++++++---- .../load/pipeline/test_filesystem_pipeline.py | 17 +++++--- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/tests/load/filesystem/test_filesystem_client.py b/tests/load/filesystem/test_filesystem_client.py index 9948e26882..f99226a7b2 100644 --- a/tests/load/filesystem/test_filesystem_client.py +++ b/tests/load/filesystem/test_filesystem_client.py @@ -11,6 +11,7 @@ FilesystemDestinationClientConfiguration, ) +from dlt.destinations.path_utils import create_path from tests.load.filesystem.utils import perform_load from tests.utils import clean_test_storage, init_test_logging from tests.utils import preserve_environ, autouse_test_storage @@ -93,6 +94,7 @@ def test_replace_write_disposition(layout: str, default_buckets_env: str) -> Non os.environ["DESTINATION__FILESYSTEM__LAYOUT"] = layout else: os.environ.pop("DESTINATION__FILESYSTEM__LAYOUT", None) + dataset_name = "test_" + uniq_id() # NOTE: context manager will delete the dataset at the end so keep it open until the end with perform_load(dataset_name, NORMALIZED_FILES, write_disposition="replace") as load_info: @@ -102,8 +104,14 @@ def test_replace_write_disposition(layout: str, default_buckets_env: str) -> Non # this path will be kept after replace job_2_load_1_path = posixpath.join( root_path, - LoadFilesystemJob.make_destination_filename( - layout, NORMALIZED_FILES[1], client.schema.name, load_id1 + create_path( + layout, + NORMALIZED_FILES[1], + client.schema.name, + load_id1, + current_datetime=client.config.current_datetime, + datetime_format=client.config.datetime_format, + extra_placeholders=client.config.extra_placeholders, ), ) @@ -115,8 +123,14 @@ def test_replace_write_disposition(layout: str, default_buckets_env: str) -> Non # this one we expect to be replaced with job_1_load_2_path = posixpath.join( root_path, - LoadFilesystemJob.make_destination_filename( - layout, NORMALIZED_FILES[0], client.schema.name, load_id2 + create_path( + layout, + NORMALIZED_FILES[0], + client.schema.name, + load_id2, + current_datetime=client.config.current_datetime, + datetime_format=client.config.datetime_format, + extra_placeholders=client.config.extra_placeholders, ), ) @@ -147,13 +161,24 @@ def test_append_write_disposition(layout: str, default_buckets_env: str) -> None client, jobs2, root_path, load_id2 = load_info layout = client.config.layout expected_files = [ - LoadFilesystemJob.make_destination_filename( - layout, job.file_name(), client.schema.name, load_id1 + create_path( + layout, + job.file_name(), + client.schema.name, + current_datetime=client.config.current_datetime, + datetime_format=client.config.datetime_format, + extra_placeholders=client.config.extra_placeholders, ) for job in jobs1 ] + [ - LoadFilesystemJob.make_destination_filename( - layout, job.file_name(), client.schema.name, load_id2 + create_path( + layout, + job.file_name(), + client.schema.name, + load_id2, + current_datetime=client.config.current_datetime, + datetime_format=client.config.datetime_format, + extra_placeholders=client.config.extra_placeholders, ) for job in jobs2 ] diff --git a/tests/load/pipeline/test_filesystem_pipeline.py b/tests/load/pipeline/test_filesystem_pipeline.py index 8fc4adc0c3..73091b4400 100644 --- a/tests/load/pipeline/test_filesystem_pipeline.py +++ b/tests/load/pipeline/test_filesystem_pipeline.py @@ -1,12 +1,14 @@ +import os import posixpath from pathlib import Path -import dlt, os +import dlt from dlt.common.utils import uniq_id from dlt.common.storages.load_storage import LoadJobInfo -from dlt.destinations.impl.filesystem.filesystem import FilesystemClient, LoadFilesystemJob +from dlt.destinations.impl.filesystem.filesystem import FilesystemClient from dlt.common.schema.typing import LOADS_TABLE_NAME +from dlt.destinations.path_utils import create_path from tests.utils import skip_if_not_active skip_if_not_active("filesystem") @@ -18,9 +20,14 @@ def assert_file_matches( """Verify file contents of load job are identical to the corresponding file in destination""" local_path = Path(job.file_path) filename = local_path.name - - destination_fn = LoadFilesystemJob.make_destination_filename( - layout, filename, client.schema.name, load_id + destination_fn = create_path( + layout, + filename, + client.schema.name, + load_id, + current_datetime=client.config.current_datetime, + datetime_format=client.config.datetime_format, + extra_placeholders=client.config.extra_placeholders, ) destination_path = posixpath.join(client.dataset_path, destination_fn)