From ff03e9e0621fe8c75a33af4954728e4c20a1870b Mon Sep 17 00:00:00 2001 From: Sultan Iman Date: Tue, 2 Apr 2024 17:29:31 +0200 Subject: [PATCH] Adjust tests --- .../load/filesystem/test_filesystem_client.py | 22 +++++-------------- .../load/pipeline/test_filesystem_pipeline.py | 14 +++++++----- 2 files changed, 14 insertions(+), 22 deletions(-) diff --git a/tests/load/filesystem/test_filesystem_client.py b/tests/load/filesystem/test_filesystem_client.py index 9948e26882..18149a685f 100644 --- a/tests/load/filesystem/test_filesystem_client.py +++ b/tests/load/filesystem/test_filesystem_client.py @@ -6,10 +6,8 @@ from dlt.common.utils import digest128, uniq_id from dlt.common.storages import FileStorage, ParsedLoadJobFileName -from dlt.destinations.impl.filesystem.filesystem import ( - LoadFilesystemJob, - FilesystemDestinationClientConfiguration, -) +from dlt.destinations.impl.filesystem.filesystem import FilesystemDestinationClientConfiguration +from dlt.destinations.impl.filesystem.layout import make_filename from tests.load.filesystem.utils import perform_load from tests.utils import clean_test_storage, init_test_logging @@ -102,9 +100,7 @@ def test_replace_write_disposition(layout: str, default_buckets_env: str) -> Non # this path will be kept after replace job_2_load_1_path = posixpath.join( root_path, - LoadFilesystemJob.make_destination_filename( - layout, NORMALIZED_FILES[1], client.schema.name, load_id1 - ), + make_filename(client.config, NORMALIZED_FILES[1], client.schema.name, load_id1), ) with perform_load( @@ -115,9 +111,7 @@ def test_replace_write_disposition(layout: str, default_buckets_env: str) -> Non # this one we expect to be replaced with job_1_load_2_path = posixpath.join( root_path, - LoadFilesystemJob.make_destination_filename( - layout, NORMALIZED_FILES[0], client.schema.name, load_id2 - ), + make_filename(client.config, NORMALIZED_FILES[0], client.schema.name, load_id2), ) # First file from load1 remains, second file is replaced by load2 @@ -147,14 +141,10 @@ def test_append_write_disposition(layout: str, default_buckets_env: str) -> None client, jobs2, root_path, load_id2 = load_info layout = client.config.layout expected_files = [ - LoadFilesystemJob.make_destination_filename( - layout, job.file_name(), client.schema.name, load_id1 - ) + make_filename(client.config, job.file_name(), client.schema.name, load_id1) for job in jobs1 ] + [ - LoadFilesystemJob.make_destination_filename( - layout, job.file_name(), client.schema.name, load_id2 - ) + make_filename(client.config, job.file_name(), client.schema.name, load_id2) for job in jobs2 ] expected_files = sorted([posixpath.join(root_path, fn) for fn in expected_files]) diff --git a/tests/load/pipeline/test_filesystem_pipeline.py b/tests/load/pipeline/test_filesystem_pipeline.py index 8fc4adc0c3..a8e00c6737 100644 --- a/tests/load/pipeline/test_filesystem_pipeline.py +++ b/tests/load/pipeline/test_filesystem_pipeline.py @@ -4,24 +4,26 @@ import dlt, os from dlt.common.utils import uniq_id from dlt.common.storages.load_storage import LoadJobInfo -from dlt.destinations.impl.filesystem.filesystem import FilesystemClient, LoadFilesystemJob +from dlt.destinations.impl.filesystem.configuration import FilesystemDestinationClientConfiguration +from dlt.destinations.impl.filesystem.filesystem import FilesystemClient +from dlt.destinations.impl.filesystem.layout import make_filename from dlt.common.schema.typing import LOADS_TABLE_NAME - from tests.utils import skip_if_not_active skip_if_not_active("filesystem") def assert_file_matches( - layout: str, job: LoadJobInfo, load_id: str, client: FilesystemClient + config: FilesystemDestinationClientConfiguration, + job: LoadJobInfo, + load_id: str, + client: FilesystemClient, ) -> None: """Verify file contents of load job are identical to the corresponding file in destination""" local_path = Path(job.file_path) filename = local_path.name - destination_fn = LoadFilesystemJob.make_destination_filename( - layout, filename, client.schema.name, load_id - ) + destination_fn = make_filename(config.layout, filename, client.schema.name, load_id) destination_path = posixpath.join(client.dataset_path, destination_fn) assert local_path.read_bytes() == client.fs_client.read_bytes(destination_path)