Skip to content

Commit

Permalink
Move defaults before update statements (#814)
Browse files Browse the repository at this point in the history
Signed-off-by: Marcel Coetzee <[email protected]>
  • Loading branch information
Pipboyguy committed Dec 26, 2023
1 parent b60436e commit 99af17e
Showing 1 changed file with 3 additions and 4 deletions.
7 changes: 3 additions & 4 deletions dlt/common/storages/fsspec_filesystem.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,17 +89,16 @@ def fsspec_from_config(config: FilesystemConfiguration) -> Tuple[AbstractFileSys
"""
proto = config.protocol
fs_kwargs: DictStrAny = {}

# defaults
fs_kwargs: DictStrAny = {"use_listings_cache": False}

if config.kwargs is not None:
fs_kwargs.update(config.kwargs)

if config.client_kwargs is not None:
fs_kwargs["client_kwargs"] = config.client_kwargs

# https://github.com/dlt-hub/dlt/issues/814#issuecomment-1867004844
fs_kwargs["use_listings_cache"] = False

if proto == "s3":
# we are careful not to override the client_kwargs key when updating fs_kwargs
s3_credentials = cast(AwsCredentials, config.credentials).to_s3fs_credentials()
Expand Down

0 comments on commit 99af17e

Please sign in to comment.