diff --git a/dlt/common/configuration/specs/gcp_credentials.py b/dlt/common/configuration/specs/gcp_credentials.py index 592c32333f..431f35c8d0 100644 --- a/dlt/common/configuration/specs/gcp_credentials.py +++ b/dlt/common/configuration/specs/gcp_credentials.py @@ -27,7 +27,9 @@ class GcpCredentials(CredentialsConfiguration): project_id: str = None - location: str = ( # DEPRECATED! and present only for backward compatibility. please set bigquery location in BigQuery configuration + location: ( + str + ) = ( # DEPRECATED! and present only for backward compatibility. please set bigquery location in BigQuery configuration "US" ) diff --git a/dlt/common/storages/fsspec_filesystem.py b/dlt/common/storages/fsspec_filesystem.py index 2d7ed0d67d..bfea605ee2 100644 --- a/dlt/common/storages/fsspec_filesystem.py +++ b/dlt/common/storages/fsspec_filesystem.py @@ -81,21 +81,14 @@ def fsspec_filesystem( ) -def fsspec_from_config(config: FilesystemConfiguration) -> Tuple[AbstractFileSystem, str]: - """Instantiates an authenticated fsspec `FileSystem` from `config` argument. - - Authenticates the following filesystems: - * s3 - * az, abfs - * gcs, gs +def prepare_fsspec_args(config: FilesystemConfiguration) -> DictStrAny: + """Prepare arguments for fsspec filesystem constructor. - Additional fsspec filesystem arguments and client arguments are gathered from the - FilesystemConfiguration object and passed to the `url_to_fs` factory. - - All other filesystems are not authenticated - - Returns: (fsspec filesystem, normalized url) + Args: + config (FilesystemConfiguration): The filesystem configuration. + Returns: + DictStrAny: The arguments for the fsspec filesystem constructor. """ proto = config.protocol fs_kwargs: DictStrAny = {"use_listings_cache": False} @@ -110,7 +103,22 @@ def fsspec_from_config(config: FilesystemConfiguration) -> Tuple[AbstractFileSys fs_kwargs["client_kwargs"].update(credentials.pop("client_kwargs")) fs_kwargs.update(credentials) + return fs_kwargs + +def fsspec_from_config(config: FilesystemConfiguration) -> Tuple[AbstractFileSystem, str]: + """Instantiates an authenticated fsspec `FileSystem` from `config` argument. + + Authenticates following filesystems: + * s3 + * az, abfs + * gcs, gs + + All other filesystems are not authenticated + + Returns: (fsspec filesystem, normalized url) + """ + fs_kwargs = prepare_fsspec_args(config) try: return url_to_fs(config.bucket_url, **fs_kwargs) # type: ignore except ModuleNotFoundError as e: