Skip to content

Commit

Permalink
add dataset type selection test and fix bug in tests
Browse files Browse the repository at this point in the history
  • Loading branch information
sh-rp committed Nov 26, 2024
1 parent 1fb17e0 commit f19a98d
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 2 deletions.
1 change: 1 addition & 0 deletions dlt/destinations/dataset/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ def __call__(self, query: Any) -> ReadableDBAPIRelation:

def table(self, table_name: str) -> SupportsReadableRelation:
# we can create an ibis powered relation if ibis is available
print(table_name in self.schema.tables, self._dataset_type)
if table_name in self.schema.tables and self._dataset_type in ("auto", "ibis"):
try:
from dlt.common.libs.ibis import create_unbound_ibis_table
Expand Down
22 changes: 20 additions & 2 deletions tests/load/test_read_interfaces.py
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,24 @@ def double_items():
)


@pytest.mark.no_load
@pytest.mark.essential
@pytest.mark.parametrize(
"populated_pipeline",
configs,
indirect=True,
ids=lambda x: x.name,
)
def test_explicit_dataset_type_selection(populated_pipeline: Pipeline):
from dlt.destinations.dataset.dataset import ReadableDBAPIRelation
from dlt.destinations.dataset.ibis_relation import ReadableIbisRelation

assert isinstance(
populated_pipeline._dataset(dataset_type="default").items, ReadableDBAPIRelation
)
assert isinstance(populated_pipeline._dataset(dataset_type="ibis").items, ReadableIbisRelation)


@pytest.mark.no_load
@pytest.mark.essential
@pytest.mark.parametrize(
Expand Down Expand Up @@ -272,7 +290,7 @@ def test_db_cursor_access(populated_pipeline: Pipeline) -> None:
)
def test_hint_preservation(populated_pipeline: Pipeline) -> None:
# NOTE: for now hints are only preserved for the default dataset
table_relationship = populated_pipeline._dataset("default").items
table_relationship = populated_pipeline._dataset(dataset_type="default").items
# check that hints are carried over to arrow table
expected_decimal_precision = 10
expected_decimal_precision_2 = 12
Expand Down Expand Up @@ -365,7 +383,7 @@ def test_limit_and_head(populated_pipeline: Pipeline) -> None:
ids=lambda x: x.name,
)
def test_column_selection(populated_pipeline: Pipeline) -> None:
table_relationship = populated_pipeline._dataset("default").items
table_relationship = populated_pipeline._dataset(dataset_type="default").items
columns = ["_dlt_load_id", "other_decimal"]
data_frame = table_relationship.select(*columns).head().df()
assert [v.lower() for v in data_frame.columns.values] == columns
Expand Down

0 comments on commit f19a98d

Please sign in to comment.