tests columns merge

dlt-hub · May 13, 2024 · 25a13bf · 25a13bf
1 parent 7f39747
commit 25a13bf
Show file tree

Hide file tree

Showing 2 changed files with 89 additions and 22 deletions.
diff --git a/tests/common/schema/test_merges.py b/tests/common/schema/test_merges.py
@@ -220,6 +220,69 @@ def test_none_resets_on_merge_column() -> None:
     assert col_a == {"name": "col1", "x-prop": None}
 
 
+def test_merge_columns() -> None:
+    columns = utils.merge_columns({"test": deepcopy(COL_1_HINTS)}, {"test_2": COL_2_HINTS})
+    # new columns added ad the end
+    assert list(columns.keys()) == ["test", "test_2"]
+    assert columns["test"] == COL_1_HINTS
+    assert columns["test_2"] == COL_2_HINTS
+
+    # replace test with new test
+    columns = utils.merge_columns(
+        {"test": deepcopy(COL_1_HINTS)}, {"test": COL_1_HINTS_NO_DEFAULTS}, merge_columns=False
+    )
+    assert list(columns.keys()) == ["test"]
+    assert columns["test"] == COL_1_HINTS_NO_DEFAULTS
+
+    # merge
+    columns = utils.merge_columns(
+        {"test": deepcopy(COL_1_HINTS)}, {"test": COL_1_HINTS_NO_DEFAULTS}, merge_columns=True
+    )
+    assert list(columns.keys()) == ["test"]
+    assert columns["test"] == utils.merge_column(deepcopy(COL_1_HINTS), COL_1_HINTS_NO_DEFAULTS)
+
+
+def test_merge_incomplete_columns() -> None:
+    incomplete_col_1 = deepcopy(COL_1_HINTS)
+    del incomplete_col_1["data_type"]
+    incomplete_col_1_nd = deepcopy(COL_1_HINTS_NO_DEFAULTS)
+    del incomplete_col_1_nd["data_type"]
+    complete_col_2 = deepcopy(COL_2_HINTS)
+    complete_col_2["data_type"] = "text"
+
+    # new incomplete added
+    columns = utils.merge_columns({"test": deepcopy(incomplete_col_1)}, {"test_2": COL_2_HINTS})
+    # new columns added ad the end
+    assert list(columns.keys()) == ["test", "test_2"]
+    assert columns["test"] == incomplete_col_1
+    assert columns["test_2"] == COL_2_HINTS
+
+    # incomplete merged with complete goes at the end
+    columns = utils.merge_columns(
+        {"test": deepcopy(incomplete_col_1), "test_2": COL_2_HINTS},
+        {"test": COL_1_HINTS_NO_DEFAULTS},
+    )
+    assert list(columns.keys()) == ["test_2", "test"]
+    assert columns["test"] == COL_1_HINTS_NO_DEFAULTS
+    assert columns["test_2"] == COL_2_HINTS
+
+    columns = utils.merge_columns(
+        {"test": deepcopy(incomplete_col_1), "test_2": COL_2_HINTS},
+        {"test": COL_1_HINTS_NO_DEFAULTS},
+        merge_columns=True,
+    )
+    assert list(columns.keys()) == ["test_2", "test"]
+    assert columns["test"] == utils.merge_column(deepcopy(COL_1_HINTS), COL_1_HINTS_NO_DEFAULTS)
+
+    # incomplete with incomplete
+    columns = utils.merge_columns(
+        {"test": deepcopy(incomplete_col_1), "test_2": COL_2_HINTS}, {"test": incomplete_col_1_nd}
+    )
+    assert list(columns.keys()) == ["test", "test_2"]
+    assert columns["test"] == incomplete_col_1_nd
+    assert columns["test_2"] == COL_2_HINTS
+
+
 def test_diff_tables() -> None:
     table: TTableSchema = {  # type: ignore[typeddict-unknown-key]
         "name": "table",

diff --git a/tests/common/test_validation.py b/tests/common/test_validation.py
@@ -283,29 +283,33 @@ def test_nested_union(test_doc: TTestRecord) -> None:
     assert e.value.value == "blah"
 
 
-def test_typeddict_friendly_exceptions() -> None:
-    class IncrementalArgs(TypedDict, total=False):
-        cursor_path: str
-        initial_value: Optional[str]
-        primary_key: Optional[TTableHintTemplate[TColumnNames]]
-        end_value: Optional[str]
-        row_order: Optional[TSortOrder]
-
-    class IncrementalConfig(IncrementalArgs, total=False):
-        start_param: str
-        end_param: Optional[str]
-
-    class Endpoint(TypedDict, total=False):
-        path: Optional[str]
-        params: Optional[Dict[str, Any]]
-        json: Optional[Dict[str, Any]]
-        data_selector: Optional[jsonpath.TJsonPath]
-        incremental: Optional[IncrementalConfig]
-
-    class EndpointResource(TypedDict, total=False):
-        endpoint: Optional[Union[str, Endpoint]]
-        write_disposition: Optional[TTableHintTemplate[TWriteDispositionConfig]]
+class IncrementalArgs(TypedDict, total=False):
+    cursor_path: str
+    initial_value: Optional[str]
+    primary_key: Optional[TTableHintTemplate[TColumnNames]]
+    end_value: Optional[str]
+    row_order: Optional[TSortOrder]
+
+
+class IncrementalConfig(IncrementalArgs, total=False):
+    start_param: str
+    end_param: Optional[str]
+
+
+class Endpoint(TypedDict, total=False):
+    path: Optional[str]
+    params: Optional[Dict[str, Any]]
+    json: Optional[Dict[str, Any]]
+    data_selector: Optional[jsonpath.TJsonPath]
+    incremental: Optional[IncrementalConfig]
 
+
+class EndpointResource(TypedDict, total=False):
+    endpoint: Optional[Union[str, Endpoint]]
+    write_disposition: Optional[TTableHintTemplate[TWriteDispositionConfig]]
+
+
+def test_typeddict_friendly_exceptions() -> None:
     valid_dict = {
         "endpoint": {
             "path": "/path",