Skip to content

Commit

Permalink
Fix flaky sorting in arrow loading test
Browse files Browse the repository at this point in the history
  • Loading branch information
steinitzu committed Nov 6, 2023
1 parent 360f165 commit 835c49c
Showing 1 changed file with 6 additions and 2 deletions.
8 changes: 6 additions & 2 deletions tests/load/pipeline/test_arrow_loading.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ def some_data():
assert some_table_columns["time"]["data_type"] == "time"

qual_name = pipeline.sql_client().make_qualified_table_name("some_data")
rows = [list(row) for row in select_data(pipeline, f"SELECT * FROM {qual_name} ORDER BY 1")]
rows = [list(row) for row in select_data(pipeline, f"SELECT * FROM {qual_name}")]

for row in rows:
for i in range(len(row)):
Expand All @@ -66,7 +66,7 @@ def some_data():
if isinstance(row[i], datetime):
row[i] = pendulum.instance(row[i])

expected = sorted([list(r.values()) for r in records], key=lambda x: x[0])
expected = sorted([list(r.values()) for r in records])

for row in expected:
for i in range(len(row)):
Expand All @@ -75,6 +75,10 @@ def some_data():

load_id = load_info.loads_ids[0]

# Sort rows by all columns except _dlt_id/_dlt_load_id for deterministic comparison
rows = sorted(rows, key=lambda row: row[:-2])
expected = sorted(expected)

for row, expected_row in zip(rows, expected):
# Compare without _dlt_id/_dlt_load_id columns
assert row[:-2] == expected_row
Expand Down

0 comments on commit 835c49c

Please sign in to comment.