Format (#814)

Signed-off-by: Marcel Coetzee <[email protected]>

dlt/common/storages/configuration.py

@@ -32,6 +32,7 @@ class SchemaStorageConfiguration(BaseConfiguration):
     )
 
     if TYPE_CHECKING:
+
         def __init__(
             self,
             schema_volume_path: str = None,
@@ -45,6 +46,7 @@ class NormalizeStorageConfiguration(BaseConfiguration):
     normalize_volume_path: str = None  # path to volume where normalized loader files will be stored
 
     if TYPE_CHECKING:
+
         def __init__(self, normalize_volume_path: str = None) -> None: ...
 
 
@@ -58,6 +60,7 @@ class LoadStorageConfiguration(BaseConfiguration):
     )
 
     if TYPE_CHECKING:
+
         def __init__(
             self, load_volume_path: str = None, delete_completed_jobs: bool = None
         ) -> None: ...
@@ -96,7 +99,6 @@ class FilesystemConfiguration(BaseConfiguration):
     kwargs: Optional[DictStrAny] = None
     client_kwargs: Optional[DictStrAny] = None
 
-
     @property
     def protocol(self) -> str:
         """`bucket_url` protocol"""
@@ -107,7 +109,6 @@ def protocol(self) -> str:
         else:
             return url.scheme
 
-
     def on_resolved(self) -> None:
         url = urlparse(self.bucket_url)
         if not url.path and not url.netloc:
@@ -120,18 +121,15 @@ def on_resolved(self) -> None:
             url = url._replace(scheme="file")
             self.bucket_url = url.geturl()
 
-
     @resolve_type("credentials")
     def resolve_credentials_type(self) -> Type[CredentialsConfiguration]:
         # use known credentials or empty credentials for unknown protocol
         return self.PROTOCOL_CREDENTIALS.get(self.protocol) or Optional[CredentialsConfiguration]  # type: ignore[return-value]
 
-
     def fingerprint(self) -> str:
         """Returns a fingerprint of bucket_url"""
         return digest128(self.bucket_url) if self.bucket_url else ""
 
-
     def __str__(self) -> str:
         """Return displayable destination location"""
         url = urlparse(self.bucket_url)
@@ -143,8 +141,12 @@ def __str__(self) -> str:
             return url._replace(netloc=new_netloc).geturl()
         return self.bucket_url
 
-
     if TYPE_CHECKING:
-        def __init__(self, bucket_url: str, credentials: FileSystemCredentials = None,
-                     kwargs: Optional[DictStrAny] = None,
-                     client_kwargs: Optional[DictStrAny] = None) -> None: ...
+
+        def __init__(
+            self,
+            bucket_url: str,
+            credentials: FileSystemCredentials = None,
+            kwargs: Optional[DictStrAny] = None,
+            client_kwargs: Optional[DictStrAny] = None,
+        ) -> None: ...

dlt/common/storages/fsspec_filesystem.py

@@ -115,7 +115,6 @@ def fsspec_from_config(config: FilesystemConfiguration) -> Tuple[AbstractFileSys
 class FileItemDict(DictStrAny):
     """A FileItem dictionary with additional methods to get fsspec filesystem, open and read files."""
 
-
     def __init__(
         self,
         mapping: FileItem,
@@ -131,7 +130,6 @@ def __init__(
         self.credentials = credentials
         super().__init__(**mapping)
 
-
     @property
     def fsspec(self) -> AbstractFileSystem:
         """The filesystem client is based on the given credentials.
@@ -144,7 +142,6 @@ def fsspec(self) -> AbstractFileSystem:
         else:
             return fsspec_filesystem(self["file_url"], self.credentials)[0]
 
-
     def open(self, mode: str = "rb", **kwargs: Any) -> IO[Any]:  # noqa: A003
         """Open the file as a fsspec file.
 
@@ -177,7 +174,6 @@ def open(self, mode: str = "rb", **kwargs: Any) -> IO[Any]:  # noqa: A003
             opened_file = self.fsspec.open(self["file_url"], mode=mode, **kwargs)
         return opened_file
 
-
     def read_bytes(self) -> bytes:
         """Read the file content.
 

tests/load/filesystem/test_filesystem_common.py

@@ -29,7 +29,12 @@ def test_filesystem_configuration() -> None:
         == Union[AzureCredentialsWithoutDefaults, AzureCredentials]
     )
     # make sure that only bucket_url and credentials are there
-    assert dict(config) == {"bucket_url": "az://root", "credentials": None, "client_kwargs": None, "kwargs": None}
+    assert dict(config) == {
+        "bucket_url": "az://root",
+        "credentials": None,
+        "client_kwargs": None,
+        "kwargs": None,
+    }
 
 
 def test_filesystem_instance(all_buckets_env: str) -> None:
@@ -96,13 +101,20 @@ def test_filesystem_instance_from_s3_endpoint(environment: Dict[str, str]) -> No
 
 
 def test_filesystem_configuration_with_additional_arguments() -> None:
-    config = FilesystemConfiguration(bucket_url="az://root", kwargs={'use_ssl': True},
-                                     client_kwargs={'verify': 'public.crt'})
-    assert dict(config) == {"bucket_url": "az://root", "credentials": None,
-                            "kwargs": {'use_ssl': True}, "client_kwargs": {'verify': 'public.crt'}}
+    config = FilesystemConfiguration(
+        bucket_url="az://root", kwargs={"use_ssl": True}, client_kwargs={"verify": "public.crt"}
+    )
+    assert dict(config) == {
+        "bucket_url": "az://root",
+        "credentials": None,
+        "kwargs": {"use_ssl": True},
+        "client_kwargs": {"verify": "public.crt"},
+    }
 
 
-def test_filesystem_instance_from_s3_endpoint_with_additional_arguments(environment: Dict[str, str]) -> None:
+def test_filesystem_instance_from_s3_endpoint_with_additional_arguments(
+    environment: Dict[str, str]
+) -> None:
     """Test that fsspec instance is correctly configured when using endpoint URL, along with additional arguments."""
     from s3fs import S3FileSystem
 
@@ -111,18 +123,27 @@ def test_filesystem_instance_from_s3_endpoint_with_additional_arguments(environm
     environment["CREDENTIALS__AWS_ACCESS_KEY_ID"] = "fake-access-key"
     environment["CREDENTIALS__AWS_SECRET_ACCESS_KEY"] = "fake-secret-key"
 
-    config = get_config(FilesystemConfiguration(bucket_url="az://root", kwargs={'use_ssl': True},
-                                                client_kwargs={'verify': 'public.crt'}))
+    config = get_config(
+        FilesystemConfiguration(
+            bucket_url="az://root", kwargs={"use_ssl": True}, client_kwargs={"verify": "public.crt"}
+        )
+    )
 
     filesystem, bucket_name = fsspec_from_config(config)
 
     assert isinstance(filesystem, S3FileSystem)
 
-    assert hasattr(filesystem, 'use_ssl'), "use_ssl additional property does not exist in filesystem instance"
+    assert hasattr(
+        filesystem, "use_ssl"
+    ), "use_ssl additional property does not exist in filesystem instance"
     assert filesystem.use_ssl, "use_ssl property does not match expected value"
 
-    assert hasattr(filesystem, 'client_kwargs'), "client_kwargs property does not exist in filesystem instance"
-    assert filesystem.client_kwargs == {'verify': 'public.crt'}, "client_kwargs property does not match expected value"
+    assert hasattr(
+        filesystem, "client_kwargs"
+    ), "client_kwargs property does not exist in filesystem instance"
+    assert filesystem.client_kwargs == {
+        "verify": "public.crt"
+    }, "client_kwargs property does not match expected value"
 
 
 def test_s3_wrong_certificate(environment: Dict[str, str]) -> None: