Skip to content

Commit

Permalink
Introduce new config fields to filesystem destination configuration
Browse files Browse the repository at this point in the history
  • Loading branch information
sultaniman committed Apr 3, 2024
1 parent 73e8176 commit 921f19a
Show file tree
Hide file tree
Showing 2 changed files with 30 additions and 1 deletion.
19 changes: 18 additions & 1 deletion dlt/destinations/impl/filesystem/configuration.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,34 @@
import dataclasses
from typing import Final, Type, Optional
from typing import Callable, Dict, Final, Optional, Type, Union

from dlt.common.configuration import configspec, resolve_type
from dlt.common.destination.reference import (
CredentialsConfiguration,
DestinationClientStagingConfiguration,
)

from dlt.common.storages import FilesystemConfiguration
from pendulum.datetime import DateTime
from typing_extensions import TypeAlias

TCurrentDatetimeCallback: TypeAlias = Callable[[], DateTime]
"""A callback which should return current datetime"""

TDatetimeFormat: TypeAlias = str
"""Datetime format or formatter callback"""

TLayoutParamCallback: TypeAlias = Callable[[str, str, str, str, str, DateTime], str]
"""A callback which should return prepared string value the following arguments passed
`schema name`, `table name`, `load_id`, `file_id`, `extension` and `current_datetime`.
"""


@configspec
class FilesystemDestinationClientConfiguration(FilesystemConfiguration, DestinationClientStagingConfiguration): # type: ignore[misc]
destination_type: Final[str] = dataclasses.field(default="filesystem", init=False, repr=False, compare=False) # type: ignore
current_datetime: Optional[Union[DateTime, TCurrentDatetimeCallback]] = None
datetime_format: Optional[TDatetimeFormat] = None
extra_placeholders: Optional[Dict[str, Union[str, TLayoutParamCallback]]] = None

@resolve_type("credentials")
def resolve_credentials_type(self) -> Type[CredentialsConfiguration]:
Expand Down
12 changes: 12 additions & 0 deletions dlt/destinations/impl/filesystem/factory.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import typing as t

import pendulum

from dlt.destinations.impl.filesystem.configuration import FilesystemDestinationClientConfiguration
from dlt.destinations.impl.filesystem import capabilities
from dlt.common.destination import Destination, DestinationCapabilitiesContext
Expand Down Expand Up @@ -55,3 +57,13 @@ def __init__(
environment=environment,
**kwargs,
)

def configuration(
self, initial_config: FilesystemDestinationClientConfiguration
) -> FilesystemDestinationClientConfiguration:
# If current_datetime is not provided then
# we need to make sure we have it one per
# filesystem destination instance
if not initial_config.current_datetime:
initial_config.current_datetime = pendulum.now()
return super().configuration(initial_config)

0 comments on commit 921f19a

Please sign in to comment.