fixes glob pattern

dlt-hub · Oct 9, 2023 · b9cb2f2 · b9cb2f2
1 parent 6f28c98
commit b9cb2f2
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/dlt/common/storages/fsspec_filesystem.py b/dlt/common/storages/fsspec_filesystem.py
@@ -178,7 +178,7 @@ def guess_mime_type(file_name: str) -> str:
 
 
 def glob_files(
-    fs_client: AbstractFileSystem, bucket_url: str, file_glob: str = "**/*"
+    fs_client: AbstractFileSystem, bucket_url: str, file_glob: str = "**"
 ) -> Iterator[FileItem]:
     """Get the files from the filesystem client.
 

diff --git a/tests/common/storages/utils.py b/tests/common/storages/utils.py
@@ -43,7 +43,7 @@ def assert_sample_files(all_file_items: List[FileItem], filesystem: AbstractFile
                 assert isinstance(lines[0], str)
 
     assert len(all_file_items) == 10
-    assert [item["file_name"] for item in all_file_items] == [
+    assert set([item["file_name"] for item in all_file_items]) == {
         'csv/freshman_kgs.csv',
         'csv/freshman_lbs.csv',
         'csv/mlb_players.csv',
@@ -54,4 +54,4 @@ def assert_sample_files(all_file_items: List[FileItem], filesystem: AbstractFile
         'met_csv/A803/A803_20230920.csv',
         'parquet/mlb_players.parquet',
         'sample.txt'
-    ]
+    }