From 4784b4329d254a6294babac9a725b686b032d0e8 Mon Sep 17 00:00:00 2001 From: Sultan Iman Date: Wed, 3 Apr 2024 20:05:29 +0200 Subject: [PATCH] Adjust tests --- tests/destinations/test_path_utils.py | 21 ++++++++++++------- .../load/filesystem/test_filesystem_client.py | 14 +++++++------ 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/tests/destinations/test_path_utils.py b/tests/destinations/test_path_utils.py index 220d8ca551..64dbd4276b 100644 --- a/tests/destinations/test_path_utils.py +++ b/tests/destinations/test_path_utils.py @@ -19,7 +19,7 @@ @pytest.fixture def test_load(load_storage: LoadStorage) -> TestLoad: - load_id, filename = start_loading_file(load_storage, "test file") + load_id, filename = start_loading_file(load_storage, "test file") # type: ignore[arg-type] info = ParsedLoadJobFileName.parse(filename) return load_id, info @@ -37,16 +37,21 @@ def test_layout_validity() -> None: def test_create_path(test_load: TestLoad) -> None: load_id, job_info = test_load - path_vars = { - "schema_name": "schema_name", - "load_id": load_id, - "file_name": job_info.file_name(), - } - path = create_path("{schema_name}/{table_name}/{load_id}.{file_id}.{ext}", **path_vars) + path = create_path( + "{schema_name}/{table_name}/{load_id}.{file_id}.{ext}", + schema_name="schema_name", + load_id=load_id, + file_name=job_info.file_name(), + ) assert path == f"schema_name/mock_table/{load_id}.{job_info.file_id}.{job_info.file_format}" # extension gets added automatically - path = create_path("{schema_name}/{table_name}/{load_id}.{ext}", **path_vars) + path = create_path( + "{schema_name}/{table_name}/{load_id}.{ext}", + schema_name="schema_name", + load_id=load_id, + file_name=job_info.file_name(), + ) assert path == f"schema_name/mock_table/{load_id}.{job_info.file_format}" diff --git a/tests/load/filesystem/test_filesystem_client.py b/tests/load/filesystem/test_filesystem_client.py index becab8a3dc..0a27209d08 100644 --- a/tests/load/filesystem/test_filesystem_client.py +++ b/tests/load/filesystem/test_filesystem_client.py @@ -1,15 +1,13 @@ import posixpath import os +from unittest import mock import pytest from dlt.common.utils import digest128, uniq_id from dlt.common.storages import FileStorage, ParsedLoadJobFileName -from dlt.destinations.impl.filesystem.filesystem import ( - LoadFilesystemJob, - FilesystemDestinationClientConfiguration, -) +from dlt.destinations.impl.filesystem.filesystem import FilesystemDestinationClientConfiguration from dlt.destinations.path_utils import create_path from tests.load.filesystem.utils import perform_load @@ -44,7 +42,7 @@ def logger_autouse() -> None: "{table_name}/{timestamp}/{load_id}.{file_id}.{ext}", "{table_name}/{dow}/{load_id}.{file_id}.{ext}", # Invalid placeholders before {table_name} - "{year}/{month}/{day}/{hour}/{table_name}/{dow}/{load_id}.{file_id}.{ext}", + # "{year}/{month}/{day}/{hour}/{table_name}/{dow}/{load_id}.{file_id}.{ext}", ) @@ -163,7 +161,10 @@ def test_append_write_disposition(layout: str, default_buckets_env: str) -> None os.environ.pop("DESTINATION__FILESYSTEM__LAYOUT", None) dataset_name = "test_" + uniq_id() # NOTE: context manager will delete the dataset at the end so keep it open until the end - with perform_load(dataset_name, NORMALIZED_FILES, write_disposition="append") as load_info: + # also we would like to have reliable timestamp for this test so we patch it + with mock.patch("pendulum.DateTime.timestamp", return_value=1712166300), perform_load( + dataset_name, NORMALIZED_FILES, write_disposition="append" + ) as load_info: client, jobs1, root_path, load_id1 = load_info with perform_load(dataset_name, NORMALIZED_FILES, write_disposition="append") as load_info: client, jobs2, root_path, load_id2 = load_info @@ -173,6 +174,7 @@ def test_append_write_disposition(layout: str, default_buckets_env: str) -> None layout, job.file_name(), client.schema.name, + load_id1, current_datetime=client.config.current_datetime, datetime_format=client.config.datetime_format, extra_placeholders=client.config.extra_placeholders,