From 5326396414568c00f4d4e7c09f94363b8b51574c Mon Sep 17 00:00:00 2001 From: Marcel Coetzee Date: Tue, 19 Dec 2023 23:17:27 +0200 Subject: [PATCH] Format (#814) Signed-off-by: Marcel Coetzee --- dlt/common/storages/configuration.py | 20 +++++---- dlt/common/storages/fsspec_filesystem.py | 4 -- .../load/filesystem/test_filesystem_common.py | 43 ++++++++++++++----- 3 files changed, 43 insertions(+), 24 deletions(-) diff --git a/dlt/common/storages/configuration.py b/dlt/common/storages/configuration.py index 79750e7c2e..eb5b861f95 100644 --- a/dlt/common/storages/configuration.py +++ b/dlt/common/storages/configuration.py @@ -32,6 +32,7 @@ class SchemaStorageConfiguration(BaseConfiguration): ) if TYPE_CHECKING: + def __init__( self, schema_volume_path: str = None, @@ -45,6 +46,7 @@ class NormalizeStorageConfiguration(BaseConfiguration): normalize_volume_path: str = None # path to volume where normalized loader files will be stored if TYPE_CHECKING: + def __init__(self, normalize_volume_path: str = None) -> None: ... @@ -58,6 +60,7 @@ class LoadStorageConfiguration(BaseConfiguration): ) if TYPE_CHECKING: + def __init__( self, load_volume_path: str = None, delete_completed_jobs: bool = None ) -> None: ... @@ -96,7 +99,6 @@ class FilesystemConfiguration(BaseConfiguration): kwargs: Optional[DictStrAny] = None client_kwargs: Optional[DictStrAny] = None - @property def protocol(self) -> str: """`bucket_url` protocol""" @@ -107,7 +109,6 @@ def protocol(self) -> str: else: return url.scheme - def on_resolved(self) -> None: url = urlparse(self.bucket_url) if not url.path and not url.netloc: @@ -120,18 +121,15 @@ def on_resolved(self) -> None: url = url._replace(scheme="file") self.bucket_url = url.geturl() - @resolve_type("credentials") def resolve_credentials_type(self) -> Type[CredentialsConfiguration]: # use known credentials or empty credentials for unknown protocol return self.PROTOCOL_CREDENTIALS.get(self.protocol) or Optional[CredentialsConfiguration] # type: ignore[return-value] - def fingerprint(self) -> str: """Returns a fingerprint of bucket_url""" return digest128(self.bucket_url) if self.bucket_url else "" - def __str__(self) -> str: """Return displayable destination location""" url = urlparse(self.bucket_url) @@ -143,8 +141,12 @@ def __str__(self) -> str: return url._replace(netloc=new_netloc).geturl() return self.bucket_url - if TYPE_CHECKING: - def __init__(self, bucket_url: str, credentials: FileSystemCredentials = None, - kwargs: Optional[DictStrAny] = None, - client_kwargs: Optional[DictStrAny] = None) -> None: ... + + def __init__( + self, + bucket_url: str, + credentials: FileSystemCredentials = None, + kwargs: Optional[DictStrAny] = None, + client_kwargs: Optional[DictStrAny] = None, + ) -> None: ... diff --git a/dlt/common/storages/fsspec_filesystem.py b/dlt/common/storages/fsspec_filesystem.py index b381655587..e5501898fd 100644 --- a/dlt/common/storages/fsspec_filesystem.py +++ b/dlt/common/storages/fsspec_filesystem.py @@ -115,7 +115,6 @@ def fsspec_from_config(config: FilesystemConfiguration) -> Tuple[AbstractFileSys class FileItemDict(DictStrAny): """A FileItem dictionary with additional methods to get fsspec filesystem, open and read files.""" - def __init__( self, mapping: FileItem, @@ -131,7 +130,6 @@ def __init__( self.credentials = credentials super().__init__(**mapping) - @property def fsspec(self) -> AbstractFileSystem: """The filesystem client is based on the given credentials. @@ -144,7 +142,6 @@ def fsspec(self) -> AbstractFileSystem: else: return fsspec_filesystem(self["file_url"], self.credentials)[0] - def open(self, mode: str = "rb", **kwargs: Any) -> IO[Any]: # noqa: A003 """Open the file as a fsspec file. @@ -177,7 +174,6 @@ def open(self, mode: str = "rb", **kwargs: Any) -> IO[Any]: # noqa: A003 opened_file = self.fsspec.open(self["file_url"], mode=mode, **kwargs) return opened_file - def read_bytes(self) -> bytes: """Read the file content. diff --git a/tests/load/filesystem/test_filesystem_common.py b/tests/load/filesystem/test_filesystem_common.py index 3bdfac1d1f..4fe44e7fce 100644 --- a/tests/load/filesystem/test_filesystem_common.py +++ b/tests/load/filesystem/test_filesystem_common.py @@ -29,7 +29,12 @@ def test_filesystem_configuration() -> None: == Union[AzureCredentialsWithoutDefaults, AzureCredentials] ) # make sure that only bucket_url and credentials are there - assert dict(config) == {"bucket_url": "az://root", "credentials": None, "client_kwargs": None, "kwargs": None} + assert dict(config) == { + "bucket_url": "az://root", + "credentials": None, + "client_kwargs": None, + "kwargs": None, + } def test_filesystem_instance(all_buckets_env: str) -> None: @@ -96,13 +101,20 @@ def test_filesystem_instance_from_s3_endpoint(environment: Dict[str, str]) -> No def test_filesystem_configuration_with_additional_arguments() -> None: - config = FilesystemConfiguration(bucket_url="az://root", kwargs={'use_ssl': True}, - client_kwargs={'verify': 'public.crt'}) - assert dict(config) == {"bucket_url": "az://root", "credentials": None, - "kwargs": {'use_ssl': True}, "client_kwargs": {'verify': 'public.crt'}} + config = FilesystemConfiguration( + bucket_url="az://root", kwargs={"use_ssl": True}, client_kwargs={"verify": "public.crt"} + ) + assert dict(config) == { + "bucket_url": "az://root", + "credentials": None, + "kwargs": {"use_ssl": True}, + "client_kwargs": {"verify": "public.crt"}, + } -def test_filesystem_instance_from_s3_endpoint_with_additional_arguments(environment: Dict[str, str]) -> None: +def test_filesystem_instance_from_s3_endpoint_with_additional_arguments( + environment: Dict[str, str] +) -> None: """Test that fsspec instance is correctly configured when using endpoint URL, along with additional arguments.""" from s3fs import S3FileSystem @@ -111,18 +123,27 @@ def test_filesystem_instance_from_s3_endpoint_with_additional_arguments(environm environment["CREDENTIALS__AWS_ACCESS_KEY_ID"] = "fake-access-key" environment["CREDENTIALS__AWS_SECRET_ACCESS_KEY"] = "fake-secret-key" - config = get_config(FilesystemConfiguration(bucket_url="az://root", kwargs={'use_ssl': True}, - client_kwargs={'verify': 'public.crt'})) + config = get_config( + FilesystemConfiguration( + bucket_url="az://root", kwargs={"use_ssl": True}, client_kwargs={"verify": "public.crt"} + ) + ) filesystem, bucket_name = fsspec_from_config(config) assert isinstance(filesystem, S3FileSystem) - assert hasattr(filesystem, 'use_ssl'), "use_ssl additional property does not exist in filesystem instance" + assert hasattr( + filesystem, "use_ssl" + ), "use_ssl additional property does not exist in filesystem instance" assert filesystem.use_ssl, "use_ssl property does not match expected value" - assert hasattr(filesystem, 'client_kwargs'), "client_kwargs property does not exist in filesystem instance" - assert filesystem.client_kwargs == {'verify': 'public.crt'}, "client_kwargs property does not match expected value" + assert hasattr( + filesystem, "client_kwargs" + ), "client_kwargs property does not exist in filesystem instance" + assert filesystem.client_kwargs == { + "verify": "public.crt" + }, "client_kwargs property does not match expected value" def test_s3_wrong_certificate(environment: Dict[str, str]) -> None: