diff --git a/tests/load/sources/sql_database/sql_source.py b/tests/load/sources/sql_database/sql_source.py index 91181841d9..4171df7d18 100644 --- a/tests/load/sources/sql_database/sql_source.py +++ b/tests/load/sources/sql_database/sql_source.py @@ -178,7 +178,7 @@ def _make_precision_table(table_name: str, nullable: bool) -> None: Table( "has_unsupported_types", self.metadata, - Column("unsupported_daterange_1", DATERANGE, nullable=False), + # Column("unsupported_daterange_1", DATERANGE, nullable=False), Column("supported_text", Text, nullable=False), Column("supported_int", Integer, nullable=False), Column("unsupported_array_1", ARRAY(Integer), nullable=False), @@ -298,7 +298,6 @@ def fake_messages(self, n: int = 9402) -> List[int]: def _fake_precision_data(self, table_name: str, n: int = 100, null_n: int = 0) -> None: table = self.metadata.tables[f"{self.schema}.{table_name}"] self.table_infos.setdefault(table_name, dict(row_count=n + null_n, is_view=False)) # type: ignore[call-overload] - rows = [ dict( int_col=random.randrange(-2147483648, 2147483647), @@ -313,7 +312,7 @@ def _fake_precision_data(self, table_name: str, n: int = 100, null_n: int = 0) - date_col=mimesis.Datetime().date(), time_col=mimesis.Datetime().time(), float_col=random.random(), - json_col={"data": [1, 2, 3]}, + json_col='{"data": [1, 2, 3]}', # NOTE: can we do this? bool_col=random.randint(0, 1) == 1, uuid_col=uuid4(), ) @@ -334,10 +333,9 @@ def _fake_chat_data(self, n: int = 9402) -> None: def _fake_unsupported_data(self, n: int = 100) -> None: table = self.metadata.tables[f"{self.schema}.has_unsupported_types"] self.table_infos.setdefault("has_unsupported_types", dict(row_count=n, is_view=False)) # type: ignore[call-overload] - rows = [ dict( - unsupported_daterange_1="[2020-01-01, 2020-09-01)", + # unsupported_daterange_1="[2020-01-01, 2020-09-01]", supported_text=mimesis.Text().word(), supported_int=random.randint(0, 100), unsupported_array_1=[1, 2, 3], diff --git a/tests/load/sources/sql_database/test_sql_database_source.py b/tests/load/sources/sql_database/test_sql_database_source.py index c97c5cc50e..94fb1f395e 100644 --- a/tests/load/sources/sql_database/test_sql_database_source.py +++ b/tests/load/sources/sql_database/test_sql_database_source.py @@ -61,7 +61,7 @@ def reset_os_environ(): def make_pipeline(destination_name: str) -> dlt.Pipeline: return dlt.pipeline( - pipeline_name="sql_database", + pipeline_name="sql_database" + uniq_id(), destination=destination_name, dataset_name="test_sql_pipeline_" + uniq_id(), full_refresh=False, @@ -806,17 +806,6 @@ def dummy_source(): columns = pipeline.default_schema.tables["has_unsupported_types"]["columns"] - # unsupported columns have unknown data type here - assert "unsupported_daterange_1" in columns - - # Arrow and pandas infer types in extract - if backend == "pyarrow": - assert columns["unsupported_daterange_1"]["data_type"] == "complex" - elif backend == "pandas": - assert columns["unsupported_daterange_1"]["data_type"] == "text" - else: - assert "data_type" not in columns["unsupported_daterange_1"] - pipeline.normalize() pipeline.load() @@ -831,7 +820,6 @@ def dummy_source(): if backend == "pyarrow": # TODO: duckdb writes structs as strings (not json encoded) to json columns # Just check that it has a value - assert rows[0]["unsupported_daterange_1"] assert isinstance(json.loads(rows[0]["unsupported_array_1"]), list) assert columns["unsupported_array_1"]["data_type"] == "complex" @@ -841,21 +829,11 @@ def dummy_source(): assert isinstance(rows[0]["supported_int"], int) elif backend == "sqlalchemy": # sqla value is a dataclass and is inferred as complex - assert columns["unsupported_daterange_1"]["data_type"] == "complex" assert columns["unsupported_array_1"]["data_type"] == "complex" - value = rows[0]["unsupported_daterange_1"] - assert set(json.loads(value).keys()) == {"lower", "upper", "bounds", "empty"} elif backend == "pandas": # pandas parses it as string - assert columns["unsupported_daterange_1"]["data_type"] == "text" - # Regex that matches daterange [2021-01-01, 2021-01-02) - assert re.match( - r"\[\d{4}-\d{2}-\d{2},\d{4}-\d{2}-\d{2}\)", - rows[0]["unsupported_daterange_1"], - ) - if type_adapter and reflection_level != "minimal": assert columns["unsupported_array_1"]["data_type"] == "complex"