Skip to content

Commit

Permalink
fixes subquery table adapter test
Browse files Browse the repository at this point in the history
  • Loading branch information
rudolfix committed Nov 23, 2024
1 parent a3fc777 commit 8da304a
Show file tree
Hide file tree
Showing 3 changed files with 18 additions and 16 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -93,14 +93,15 @@ table = sql_table(
You can add computed columns to the table definition by converting it into a subquery:
```py
def add_max_timestamp(table):
computed_max_timestamp = sa.func.greatest(table.c.created_at, table.c.updated_at).label('max_timestamp')
subquery = sa.select(
[table, computed_max_timestamp]
).subquery()
computed_max_timestamp = sa.sql.type_coerce(
sa.func.greatest(table.c.created_at, table.c.updated_at),
sqltypes.DateTime,
).label("max_timestamp")
subquery = sa.select(*table.c, computed_max_timestamp).subquery()
return subquery
```
We add new `max_timestamp` column that is a MIN of `created_at` and `updated_at` columns and then convert it into a subquery
because we intend to use it for incremental loading.
We add new `max_timestamp` column that is a MAX of `created_at` and `updated_at` columns and then we convert it into a subquery
because we intend to use it for incremental loading which will attach a `WHERE` clause to it.

```py
import dlt
Expand Down
10 changes: 5 additions & 5 deletions tests/common/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -407,7 +407,7 @@ def test_group_dict_of_lists_various_length_lists():
assert sizes == sorted(sizes, reverse=True), "Sizes of decomposed dicts are not decreasing"


def function_test(a, b):
def function_typeerror_exc(a, b):
raise TypeError("wrong type")


Expand All @@ -431,12 +431,12 @@ def _function_test(a, *, b=None):
assert is_typeerror_due_to_wrong_call(exc, _function_test) is False

try:
function_test() # type: ignore[call-arg]
function_typeerror_exc() # type: ignore[call-arg]
except Exception as exc:
assert is_typeerror_due_to_wrong_call(exc, function_test) is True
assert is_typeerror_due_to_wrong_call(exc, function_typeerror_exc) is True

try:
function_test("a", "b")
function_typeerror_exc("a", "b")
except Exception as exc:
assert str(exc) == "wrong type"
assert is_typeerror_due_to_wrong_call(exc, function_test) is False
assert is_typeerror_due_to_wrong_call(exc, function_typeerror_exc) is False
11 changes: 6 additions & 5 deletions tests/load/sources/sql_database/test_sql_database_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -275,14 +275,15 @@ def query_adapter(

@pytest.mark.parametrize("backend", ["sqlalchemy", "pandas", "pyarrow"])
def test_computed_column(sql_source_db: SQLAlchemySourceDB, backend: TableBackend) -> None:
from dlt.common.libs.sql_alchemy import Table, sa
from dlt.common.libs.sql_alchemy import Table, sa, sqltypes
from dlt.sources.sql_database.helpers import SelectAny

def add_max_timestamp(table: Table) -> SelectAny:
computed_max_timestamp = sa.func.greatest(table.c.created_at, table.c.updated_at).label(
"max_timestamp"
)
subquery = sa.select([table, computed_max_timestamp]).subquery()
computed_max_timestamp = sa.sql.type_coerce(
sa.func.greatest(table.c.created_at, table.c.updated_at),
sqltypes.DateTime,
).label("max_timestamp")
subquery = sa.select(*table.c, computed_max_timestamp).subquery()
return subquery

read_table = sql_table(
Expand Down

0 comments on commit 8da304a

Please sign in to comment.