Skip to content

Commit

Permalink
Enable auto_mkdir for local filesystem
Browse files Browse the repository at this point in the history
  • Loading branch information
sultaniman committed Apr 3, 2024
1 parent 4784b43 commit c58a584
Showing 1 changed file with 6 additions and 1 deletion.
7 changes: 6 additions & 1 deletion dlt/common/storages/fsspec_filesystem.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,10 @@
AzureCredentials,
)
from dlt.common.exceptions import MissingDependencyException
from dlt.common.storages.configuration import FileSystemCredentials, FilesystemConfiguration
from dlt.common.storages.configuration import (
FileSystemCredentials,
FilesystemConfiguration,
)
from dlt.common.time import ensure_pendulum_datetime
from dlt.common.typing import DictStrAny

Expand Down Expand Up @@ -144,6 +147,8 @@ def fsspec_from_config(config: FilesystemConfiguration) -> Tuple[AbstractFileSys
fs_kwargs = prepare_fsspec_args(config)

try:
if config.bucket_url.startswith("file://"):
fs_kwargs["auto_mkdir"] = True
return url_to_fs(config.bucket_url, **fs_kwargs) # type: ignore
except ModuleNotFoundError as e:
raise MissingDependencyException(
Expand Down

0 comments on commit c58a584

Please sign in to comment.