Skip to content

Commit

Permalink
add nested data test
Browse files Browse the repository at this point in the history
  • Loading branch information
IlyaFaer committed Mar 28, 2024
1 parent c239109 commit f3c5068
Showing 1 changed file with 15 additions and 7 deletions.
22 changes: 15 additions & 7 deletions tests/load/bigquery/test_bigquery_streaming_insert.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,18 @@
import dlt
from dlt.destinations.exceptions import DatabaseTransientException
from dlt.destinations.impl.bigquery.bigquery_adapter import bigquery_adapter
from tests.pipeline.utils import assert_load_info


def test_bigquery_streaming_insert():
pipe = dlt.pipeline(destination="bigquery")
pack = pipe.run([{"field1": 1, "field2": 2}], table_name="test_streaming_items")
def test_bigquery_adapter_streaming_insert():
@dlt.resource
def test_resource():
yield {"field1": 1, "field2": 2}

bigquery_adapter(test_resource, insert_api="streaming")

pipe = dlt.pipeline(pipeline_name="insert_test", destination="bigquery")
pack = pipe.run(test_resource, table_name="test_streaming_items")

assert_load_info(pack)

Expand All @@ -15,19 +22,20 @@ def test_bigquery_streaming_insert():
assert tuple(res[0])[:2] == (1, 2)


def test_bigquery_adapter_streaming_insert():
def test_bigquery_streaming_nested_data():
@dlt.resource
def test_resource():
yield {"field1": 1, "field2": 2}
yield {"field1": {"nested_field": 1}, "field2": {"nested_field": 2}}

bigquery_adapter(test_resource, insert_api="streaming")

pipe = dlt.pipeline(destination="bigquery")
pipe = dlt.pipeline(pipeline_name="insert_test", destination="bigquery")
pack = pipe.run(test_resource, table_name="test_streaming_items")

assert_load_info(pack)

with pipe.sql_client() as client:
with client.execute_query("SELECT * FROM test_streaming_items;") as cursor:
res = cursor.fetchall()
assert tuple(res[0])[:2] == (1, 2)
assert res[0]["field1__nested_field"] == 1
assert res[0]["field2__nested_field"] == 2

0 comments on commit f3c5068

Please sign in to comment.