Skip to content

Commit

Permalink
feat(filesystem): add compression flag if the read file is GZ
Browse files Browse the repository at this point in the history
  • Loading branch information
IlyaFaer committed Jan 26, 2024
1 parent 859258d commit 5522a92
Showing 1 changed file with 7 additions and 2 deletions.
9 changes: 7 additions & 2 deletions dlt/common/storages/fsspec_filesystem.py
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ def open(self, mode: str = "rb", **kwargs: Any) -> IO[Any]: # noqa: A003
IOBase: The fsspec file.
"""
opened_file: IO[Any]
# if the user has already extracted the content, we use it so there will be no need to
# if the user has already extracted the content, we use it so there is no need to
# download the file again.
if "file_content" in self:
bytes_io = BytesIO(self["file_content"])
Expand All @@ -185,7 +185,12 @@ def open(self, mode: str = "rb", **kwargs: Any) -> IO[Any]: # noqa: A003
**text_kwargs,
)
else:
opened_file = self.fsspec.open(self["file_url"], mode=mode, **kwargs)
opened_file = self.fsspec.open(
self["file_url"],
mode=mode,
compression="gzip" if self["file_url"].endswith(".gz") else None,
**kwargs,
)
return opened_file

def read_bytes(self) -> bytes:
Expand Down

0 comments on commit 5522a92

Please sign in to comment.