From b7e89f4d420f56dbdd2cdbccaf8e98bb0e3450f7 Mon Sep 17 00:00:00 2001 From: Sultan Iman Date: Wed, 7 Feb 2024 12:27:31 +0100 Subject: [PATCH] Simplify tests --- tests/pipeline/test_pipeline_extra.py | 46 ++++++--------------------- 1 file changed, 10 insertions(+), 36 deletions(-) diff --git a/tests/pipeline/test_pipeline_extra.py b/tests/pipeline/test_pipeline_extra.py index 3dc4a66505..460f847425 100644 --- a/tests/pipeline/test_pipeline_extra.py +++ b/tests/pipeline/test_pipeline_extra.py @@ -207,14 +207,7 @@ class Child(BaseModel): optional_child_attribute: Optional[str] = None -@pytest.mark.parametrize( - "destination_config", - destinations_configs(default_sql_configs=True, subset=["duckdb"]), - ids=lambda x: x.name, -) -def test_flattens_model_when_skip_complex_types_is_set( - destination_config: DestinationTestConfiguration, -) -> None: +def test_flattens_model_when_skip_complex_types_is_set() -> None: class Parent(BaseModel): child: Child optional_parent_attribute: Optional[str] = None @@ -228,16 +221,8 @@ class Parent(BaseModel): }, } - @dlt.resource - def res(): - yield [example_data] - - @dlt.source(max_table_nesting=1) - def src(): - yield res() - - p = destination_config.setup_pipeline("example", full_refresh=True) - p.run(src(), table_name="items", columns=Parent) + p = dlt.pipeline("example", destination="duckdb") + p.run([example_data], table_name="items", columns=Parent) with p.sql_client() as client: with client.execute_query("SELECT * FROM items") as cursor: @@ -246,6 +231,8 @@ def src(): for val, col in zip(cursor.fetchall()[0], cursor.description) if col[0] not in ("_dlt_id", "_dlt_load_id") } + + # Check if child dictionary is flattened and added to schema assert loaded_values == { "child__child_attribute": "any string", "child__optional_child_attribute": None, @@ -282,14 +269,7 @@ def src(): } -@pytest.mark.parametrize( - "destination_config", - destinations_configs(default_sql_configs=True, subset=["duckdb"]), - ids=lambda x: x.name, -) -def test_flattens_model_when_skip_complex_types_is_not_set( - destination_config: DestinationTestConfiguration, -): +def test_flattens_model_when_skip_complex_types_is_not_set(): class Parent(BaseModel): child: Child optional_parent_attribute: Optional[str] = None @@ -307,16 +287,8 @@ class Parent(BaseModel): }, } - @dlt.resource - def res(): - yield [example_data] - - @dlt.source(max_table_nesting=1) - def src(): - yield res() - - p = destination_config.setup_pipeline("example", full_refresh=True) - p.run(src(), table_name="items", columns=Parent) + p = dlt.pipeline("example", destination="duckdb") + p.run([example_data], table_name="items", columns=Parent) with p.sql_client() as client: with client.execute_query("SELECT * FROM items") as cursor: @@ -326,6 +298,8 @@ def src(): if col[0] not in ("_dlt_id", "_dlt_load_id") } + # Check if complex fields preserved + # their contents and were not flattened assert loaded_values == { "child": '{"child_attribute":"any string","optional_child_attribute":null}', "optional_parent_attribute": None,