Rename keywords, add class attrs (#814)

Signed-off-by: Marcel Coetzee <[email protected]>
dlt-hub · Dec 19, 2023 · 939d8ea · 939d8ea
1 parent 03e8de5
commit 939d8ea
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 7 deletions.
diff --git a/dlt/common/storages/configuration.py b/dlt/common/storages/configuration.py
@@ -12,9 +12,11 @@
     AzureCredentialsWithoutDefaults,
     BaseConfiguration,
 )
+from dlt.common.typing import DictStrAny
 from dlt.common.utils import digest128
 from dlt.common.configuration.exceptions import ConfigurationValueError
 
+
 TSchemaFileFormat = Literal["json", "yaml"]
 SchemaFileExtensions = get_args(TSchemaFileFormat)
 
@@ -30,7 +32,6 @@ class SchemaStorageConfiguration(BaseConfiguration):
     )
 
     if TYPE_CHECKING:
-
         def __init__(
             self,
             schema_volume_path: str = None,
@@ -44,7 +45,6 @@ class NormalizeStorageConfiguration(BaseConfiguration):
     normalize_volume_path: str = None  # path to volume where normalized loader files will be stored
 
     if TYPE_CHECKING:
-
         def __init__(self, normalize_volume_path: str = None) -> None: ...
 
 
@@ -58,7 +58,6 @@ class LoadStorageConfiguration(BaseConfiguration):
     )
 
     if TYPE_CHECKING:
-
         def __init__(
             self, load_volume_path: str = None, delete_completed_jobs: bool = None
         ) -> None: ...
@@ -94,6 +93,9 @@ class FilesystemConfiguration(BaseConfiguration):
     bucket_url: str = None
     # should be a union of all possible credentials as found in PROTOCOL_CREDENTIALS
     credentials: FileSystemCredentials
+    kwargs: Optional[DictStrAny] = None
+    client_kwargs: Optional[DictStrAny] = None
+
 
     @property
     def protocol(self) -> str:
@@ -105,6 +107,7 @@ def protocol(self) -> str:
         else:
             return url.scheme
 
+
     def on_resolved(self) -> None:
         url = urlparse(self.bucket_url)
         if not url.path and not url.netloc:
@@ -117,15 +120,18 @@ def on_resolved(self) -> None:
             url = url._replace(scheme="file")
             self.bucket_url = url.geturl()
 
+
     @resolve_type("credentials")
     def resolve_credentials_type(self) -> Type[CredentialsConfiguration]:
         # use known credentials or empty credentials for unknown protocol
         return self.PROTOCOL_CREDENTIALS.get(self.protocol) or Optional[CredentialsConfiguration]  # type: ignore[return-value]
 
+
     def fingerprint(self) -> str:
         """Returns a fingerprint of bucket_url"""
         return digest128(self.bucket_url) if self.bucket_url else ""
 
+
     def __str__(self) -> str:
         """Return displayable destination location"""
         url = urlparse(self.bucket_url)
@@ -137,6 +143,6 @@ def __str__(self) -> str:
             return url._replace(netloc=new_netloc).geturl()
         return self.bucket_url
 
-    if TYPE_CHECKING:
 
+    if TYPE_CHECKING:
         def __init__(self, bucket_url: str, credentials: FileSystemCredentials = None) -> None: ...
diff --git a/tests/load/filesystem/test_filesystem_common.py b/tests/load/filesystem/test_filesystem_common.py
@@ -97,17 +97,17 @@ def test_filesystem_instance_from_s3_endpoint(environment: Dict[str, str]) -> No
 
 
 def test_filesystem_configuration_with_additional_arguments() -> None:
-    config = FilesystemConfiguration(bucket_url="az://root", additional_args={'use_ssl': True},
+    config = FilesystemConfiguration(bucket_url="az://root", kwargs={'use_ssl': True},
                                      client_kwargs={'verify': 'public.crt'})
     assert dict(config) == {"bucket_url": "az://root", "credentials": None,
-                            "additional_args": {'use_ssl': True}, "client_kwargs": {'verify': 'public.crt'}}
+                            "kwargs": {'use_ssl': True}, "client_kwargs": {'verify': 'public.crt'}}
 
 
 def test_filesystem_instance_from_s3_endpoint_with_additional_arguments(environment: Dict[str, str]) -> None:
     """Test that fsspec instance is correctly configured when using endpoint URL, along with additional arguments."""
     from s3fs import S3FileSystem
 
-    config = FilesystemConfiguration(bucket_url="s3://dummy-bucket", additional_args={'use_ssl': True},
+    config = FilesystemConfiguration(bucket_url="s3://dummy-bucket", kwargs={'use_ssl': True},
                                      client_kwargs={'verify': 'public.crt'})
     filesystem, bucket_name = fsspec_from_config(config)