Skip to content

Commit

Permalink
make glob test strict on file list
Browse files Browse the repository at this point in the history
  • Loading branch information
rudolfix committed Jan 30, 2024
1 parent f4c8007 commit e6134ef
Showing 1 changed file with 6 additions and 4 deletions.
10 changes: 6 additions & 4 deletions tests/common/storages/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,16 +41,19 @@ def assert_sample_files(
"met_csv/A803/A803_20230919.csv",
"met_csv/A803/A803_20230920.csv",
"parquet/mlb_players.parquet",
"gzip/taxi.csv.gz",
"sample.txt",
}

assert len(all_file_items) >= 10
assert set([item["file_name"] for item in all_file_items]) >= minimally_expected_file_items

for item in all_file_items:
# only run tests on file items we know
if item["file_name"] not in minimally_expected_file_items:
# skip pseudo files that look like folders
if item["file_url"].endswith("/"):
continue
# only accept file items we know
assert item["file_name"] in minimally_expected_file_items

assert isinstance(item["file_name"], str)
assert item["file_url"].endswith(item["file_name"])
assert item["file_url"].startswith(config.protocol)
Expand All @@ -73,7 +76,6 @@ def assert_sample_files(
open_content = f.read()
assert content == open_content
# read via various readers
print(item)
if item["mime_type"] == "text/csv":
# parse csv
with file_dict.open(mode="rt") as f:
Expand Down

0 comments on commit e6134ef

Please sign in to comment.