Skip to content

Commit

Permalink
feat(filesystem): use only netloc and scheme for fingerprint
Browse files Browse the repository at this point in the history
  • Loading branch information
IlyaFaer committed Jun 26, 2024
1 parent 6b83cee commit 2feb807
Show file tree
Hide file tree
Showing 2 changed files with 24 additions and 7 deletions.
15 changes: 13 additions & 2 deletions dlt/common/storages/configuration.py
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,19 @@ def resolve_credentials_type(self) -> Type[CredentialsConfiguration]:
return self.PROTOCOL_CREDENTIALS.get(self.protocol) or Optional[CredentialsConfiguration] # type: ignore[return-value]

def fingerprint(self) -> str:
"""Returns a fingerprint of bucket_url"""
return digest128(self.bucket_url) if self.bucket_url else ""
"""Returns a fingerprint of bucket schema and netloc.
Returns:
str: Fingerprint.
"""
if not self.bucket_url:
return ""

if self.is_local_path(self.bucket_url):
return digest128("")

uri = urlparse(self.bucket_url)
return digest128(self.bucket_url.replace(uri.path, ""))

def __str__(self) -> str:
"""Return displayable destination location"""
Expand Down
16 changes: 11 additions & 5 deletions tests/load/filesystem/test_filesystem_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,17 @@ def logger_autouse() -> None:
]


def test_filesystem_destination_configuration() -> None:
assert FilesystemDestinationClientConfiguration().fingerprint() == ""
assert FilesystemDestinationClientConfiguration(
bucket_url="s3://cool"
).fingerprint() == digest128("s3://cool")
@pytest.mark.parametrize(
"url, exp",
(
(None, ""),
("/path/path2", digest128("")),
("s3://cool", digest128("s3://cool")),
("s3://cool.domain/path/path2", digest128("s3://cool.domain")),
),
)
def test_filesystem_destination_configuration(url, exp) -> None:
assert FilesystemDestinationClientConfiguration(bucket_url=url).fingerprint() == exp


@pytest.mark.parametrize("write_disposition", ("replace", "append", "merge"))
Expand Down

0 comments on commit 2feb807

Please sign in to comment.