From 939d8ea4794527eabef65f9bfe15d0dbe9d39919 Mon Sep 17 00:00:00 2001 From: Marcel Coetzee Date: Tue, 19 Dec 2023 21:40:59 +0200 Subject: [PATCH] Rename keywords, add class attrs (#814) Signed-off-by: Marcel Coetzee --- dlt/common/storages/configuration.py | 14 ++++++++++---- tests/load/filesystem/test_filesystem_common.py | 6 +++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/dlt/common/storages/configuration.py b/dlt/common/storages/configuration.py index 3e44948995..d03ce09522 100644 --- a/dlt/common/storages/configuration.py +++ b/dlt/common/storages/configuration.py @@ -12,9 +12,11 @@ AzureCredentialsWithoutDefaults, BaseConfiguration, ) +from dlt.common.typing import DictStrAny from dlt.common.utils import digest128 from dlt.common.configuration.exceptions import ConfigurationValueError + TSchemaFileFormat = Literal["json", "yaml"] SchemaFileExtensions = get_args(TSchemaFileFormat) @@ -30,7 +32,6 @@ class SchemaStorageConfiguration(BaseConfiguration): ) if TYPE_CHECKING: - def __init__( self, schema_volume_path: str = None, @@ -44,7 +45,6 @@ class NormalizeStorageConfiguration(BaseConfiguration): normalize_volume_path: str = None # path to volume where normalized loader files will be stored if TYPE_CHECKING: - def __init__(self, normalize_volume_path: str = None) -> None: ... @@ -58,7 +58,6 @@ class LoadStorageConfiguration(BaseConfiguration): ) if TYPE_CHECKING: - def __init__( self, load_volume_path: str = None, delete_completed_jobs: bool = None ) -> None: ... @@ -94,6 +93,9 @@ class FilesystemConfiguration(BaseConfiguration): bucket_url: str = None # should be a union of all possible credentials as found in PROTOCOL_CREDENTIALS credentials: FileSystemCredentials + kwargs: Optional[DictStrAny] = None + client_kwargs: Optional[DictStrAny] = None + @property def protocol(self) -> str: @@ -105,6 +107,7 @@ def protocol(self) -> str: else: return url.scheme + def on_resolved(self) -> None: url = urlparse(self.bucket_url) if not url.path and not url.netloc: @@ -117,15 +120,18 @@ def on_resolved(self) -> None: url = url._replace(scheme="file") self.bucket_url = url.geturl() + @resolve_type("credentials") def resolve_credentials_type(self) -> Type[CredentialsConfiguration]: # use known credentials or empty credentials for unknown protocol return self.PROTOCOL_CREDENTIALS.get(self.protocol) or Optional[CredentialsConfiguration] # type: ignore[return-value] + def fingerprint(self) -> str: """Returns a fingerprint of bucket_url""" return digest128(self.bucket_url) if self.bucket_url else "" + def __str__(self) -> str: """Return displayable destination location""" url = urlparse(self.bucket_url) @@ -137,6 +143,6 @@ def __str__(self) -> str: return url._replace(netloc=new_netloc).geturl() return self.bucket_url - if TYPE_CHECKING: + if TYPE_CHECKING: def __init__(self, bucket_url: str, credentials: FileSystemCredentials = None) -> None: ... diff --git a/tests/load/filesystem/test_filesystem_common.py b/tests/load/filesystem/test_filesystem_common.py index 560a6fb278..3d762d60ca 100644 --- a/tests/load/filesystem/test_filesystem_common.py +++ b/tests/load/filesystem/test_filesystem_common.py @@ -97,17 +97,17 @@ def test_filesystem_instance_from_s3_endpoint(environment: Dict[str, str]) -> No def test_filesystem_configuration_with_additional_arguments() -> None: - config = FilesystemConfiguration(bucket_url="az://root", additional_args={'use_ssl': True}, + config = FilesystemConfiguration(bucket_url="az://root", kwargs={'use_ssl': True}, client_kwargs={'verify': 'public.crt'}) assert dict(config) == {"bucket_url": "az://root", "credentials": None, - "additional_args": {'use_ssl': True}, "client_kwargs": {'verify': 'public.crt'}} + "kwargs": {'use_ssl': True}, "client_kwargs": {'verify': 'public.crt'}} def test_filesystem_instance_from_s3_endpoint_with_additional_arguments(environment: Dict[str, str]) -> None: """Test that fsspec instance is correctly configured when using endpoint URL, along with additional arguments.""" from s3fs import S3FileSystem - config = FilesystemConfiguration(bucket_url="s3://dummy-bucket", additional_args={'use_ssl': True}, + config = FilesystemConfiguration(bucket_url="s3://dummy-bucket", kwargs={'use_ssl': True}, client_kwargs={'verify': 'public.crt'}) filesystem, bucket_name = fsspec_from_config(config)