Skip to content

Commit

Permalink
Simplify tests
Browse files Browse the repository at this point in the history
  • Loading branch information
Sultan Iman committed Feb 7, 2024
1 parent ebe6ced commit b7e89f4
Showing 1 changed file with 10 additions and 36 deletions.
46 changes: 10 additions & 36 deletions tests/pipeline/test_pipeline_extra.py
Original file line number Diff line number Diff line change
Expand Up @@ -207,14 +207,7 @@ class Child(BaseModel):
optional_child_attribute: Optional[str] = None


@pytest.mark.parametrize(
"destination_config",
destinations_configs(default_sql_configs=True, subset=["duckdb"]),
ids=lambda x: x.name,
)
def test_flattens_model_when_skip_complex_types_is_set(
destination_config: DestinationTestConfiguration,
) -> None:
def test_flattens_model_when_skip_complex_types_is_set() -> None:
class Parent(BaseModel):
child: Child
optional_parent_attribute: Optional[str] = None
Expand All @@ -228,16 +221,8 @@ class Parent(BaseModel):
},
}

@dlt.resource
def res():
yield [example_data]

@dlt.source(max_table_nesting=1)
def src():
yield res()

p = destination_config.setup_pipeline("example", full_refresh=True)
p.run(src(), table_name="items", columns=Parent)
p = dlt.pipeline("example", destination="duckdb")
p.run([example_data], table_name="items", columns=Parent)

with p.sql_client() as client:
with client.execute_query("SELECT * FROM items") as cursor:
Expand All @@ -246,6 +231,8 @@ def src():
for val, col in zip(cursor.fetchall()[0], cursor.description)
if col[0] not in ("_dlt_id", "_dlt_load_id")
}

# Check if child dictionary is flattened and added to schema
assert loaded_values == {
"child__child_attribute": "any string",
"child__optional_child_attribute": None,
Expand Down Expand Up @@ -282,14 +269,7 @@ def src():
}


@pytest.mark.parametrize(
"destination_config",
destinations_configs(default_sql_configs=True, subset=["duckdb"]),
ids=lambda x: x.name,
)
def test_flattens_model_when_skip_complex_types_is_not_set(
destination_config: DestinationTestConfiguration,
):
def test_flattens_model_when_skip_complex_types_is_not_set():
class Parent(BaseModel):
child: Child
optional_parent_attribute: Optional[str] = None
Expand All @@ -307,16 +287,8 @@ class Parent(BaseModel):
},
}

@dlt.resource
def res():
yield [example_data]

@dlt.source(max_table_nesting=1)
def src():
yield res()

p = destination_config.setup_pipeline("example", full_refresh=True)
p.run(src(), table_name="items", columns=Parent)
p = dlt.pipeline("example", destination="duckdb")
p.run([example_data], table_name="items", columns=Parent)

with p.sql_client() as client:
with client.execute_query("SELECT * FROM items") as cursor:
Expand All @@ -326,6 +298,8 @@ def src():
if col[0] not in ("_dlt_id", "_dlt_load_id")
}

# Check if complex fields preserved
# their contents and were not flattened
assert loaded_values == {
"child": '{"child_attribute":"any string","optional_child_attribute":null}',
"optional_parent_attribute": None,
Expand Down

0 comments on commit b7e89f4

Please sign in to comment.