From 5b20c1276d1aac387044709b3931ec08f3234548 Mon Sep 17 00:00:00 2001 From: Sultan Iman Date: Wed, 3 Apr 2024 19:06:21 +0200 Subject: [PATCH] Fix failing tests --- dlt/destinations/path_utils.py | 4 +- tests/destinations/test_path_utils.py | 65 +++++++++++++++++---------- 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/dlt/destinations/path_utils.py b/dlt/destinations/path_utils.py index 197daa5620..2f07220bb0 100644 --- a/dlt/destinations/path_utils.py +++ b/dlt/destinations/path_utils.py @@ -51,7 +51,7 @@ def __init__( self.job_info = job_info self.load_id = load_id self.schema_name = schema_name - self._params: Dict[str, Any] = {} + self._params: Dict[str, Any] = {"schema_name": schema_name} self.table_name = None self.file_id = None @@ -246,7 +246,7 @@ def get_table_prefix_layout( ) with LayoutHelper(layout, extras.params) as layout_helper: # fail if table name is not defined - if "table_name" not in layout_helper.placeholders: + if "table_name" not in layout_helper.layout_placeholders: raise CantExtractTablePrefix(layout, "{table_name} placeholder not found. ") table_name_index = layout_helper.layout_placeholders.index("table_name") diff --git a/tests/destinations/test_path_utils.py b/tests/destinations/test_path_utils.py index 1cf2b17d76..6f25a43eeb 100644 --- a/tests/destinations/test_path_utils.py +++ b/tests/destinations/test_path_utils.py @@ -1,46 +1,65 @@ +from typing import Tuple + import pytest -from dlt.destinations import path_utils +from dlt.common.storages import LoadStorage +from dlt.common.storages.load_package import ParsedLoadJobFileName +from dlt.destinations.path_utils import ( + create_path, + get_table_prefix_layout, + ExtraParams, + LayoutHelper, +) from dlt.destinations.exceptions import InvalidFilesystemLayout, CantExtractTablePrefix +from tests.common.storages.utils import start_loading_file, load_storage + +extra_params = ExtraParams() + +TestLoad = Tuple[str, ParsedLoadJobFileName] + + +@pytest.fixture +def test_load(load_storage: LoadStorage) -> TestLoad: + load_id, filename = start_loading_file(load_storage, "test file") + info = ParsedLoadJobFileName.parse(filename) + return load_id, info + def test_layout_validity() -> None: - path_utils.check_layout("{table_name}") - path_utils.check_layout("{schema_name}/{table_name}/{load_id}.{file_id}.{ext}") + LayoutHelper("{table_name}", extra_params.params).check_layout() + LayoutHelper( + "{schema_name}/{table_name}/{load_id}.{file_id}.{ext}", extra_params.params + ).check_layout() with pytest.raises(InvalidFilesystemLayout) as exc: - path_utils.check_layout("{other_ph}.{table_name}") + LayoutHelper("{other_ph}.{table_name}", extra_params.params).check_layout() assert exc.value.invalid_placeholders == ["other_ph"] -def test_create_path() -> None: +def test_create_path(test_load: TestLoad) -> None: + load_id, job_info = test_load path_vars = { "schema_name": "schema_name", - "table_name": "table_name", - "load_id": "load_id", - "file_id": "file_id", - "ext": "ext", + "load_id": load_id, + "file_name": job_info.file_name(), } - path = path_utils.create_path( - "{schema_name}/{table_name}/{load_id}.{file_id}.{ext}", **path_vars - ) - assert path == "schema_name/table_name/load_id.file_id.ext" + path = create_path("{schema_name}/{table_name}/{load_id}.{file_id}.{ext}", **path_vars) + assert path == f"schema_name/mock_table/{load_id}.{job_info.file_id}.{job_info.file_format}" # extension gets added automatically - path = path_utils.create_path("{schema_name}/{table_name}/{load_id}", **path_vars) - assert path == "schema_name/table_name/load_id.ext" + path = create_path("{schema_name}/{table_name}/{load_id}.{ext}", **path_vars) + assert path == f"schema_name/mock_table/{load_id}.{job_info.file_format}" def test_get_table_prefix_layout() -> None: - prefix_layout = path_utils.get_table_prefix_layout( - "{schema_name}/{table_name}/{load_id}.{file_id}.{ext}" - ) + prefix_layout = get_table_prefix_layout("{schema_name}/{table_name}/{load_id}.{file_id}.{ext}") assert prefix_layout == "{schema_name}/{table_name}/" assert ( prefix_layout.format(schema_name="my_schema", table_name="my_table") == "my_schema/my_table/" ) - prefix_layout = path_utils.get_table_prefix_layout( + prefix_layout = get_table_prefix_layout( "some_random{schema_name}/stuff_in_between/{table_name}/{load_id}" ) assert prefix_layout == "some_random{schema_name}/stuff_in_between/{table_name}/" @@ -51,19 +70,19 @@ def test_get_table_prefix_layout() -> None: # disallow missing table_name with pytest.raises(CantExtractTablePrefix): - path_utils.get_table_prefix_layout("some_random{schema_name}/stuff_in_between/") + get_table_prefix_layout("some_random{schema_name}/stuff_in_between/") # disallow other params before table_name with pytest.raises(CantExtractTablePrefix): - path_utils.get_table_prefix_layout("{file_id}some_random{table_name}/stuff_in_between/") + get_table_prefix_layout("{file_id}some_random{table_name}/stuff_in_between/") # disallow any placeholders before table name (ie. Athena) with pytest.raises(CantExtractTablePrefix): - path_utils.get_table_prefix_layout( + get_table_prefix_layout( "{schema_name}some_random{table_name}/stuff_in_between/", supported_prefix_placeholders=[], ) # disallow table_name without following separator with pytest.raises(CantExtractTablePrefix): - path_utils.get_table_prefix_layout("{schema_name}/{table_name}{load_id}.{file_id}.{ext}") + get_table_prefix_layout("{schema_name}/{table_name}{load_id}.{file_id}.{ext}")