From cf37c6624df1d857f33ade36d76764799d04a2fb Mon Sep 17 00:00:00 2001 From: Sultan Iman Date: Wed, 3 Apr 2024 13:53:29 +0200 Subject: [PATCH] Let fsspec to create destination directory --- dlt/destinations/impl/filesystem/filesystem.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/dlt/destinations/impl/filesystem/filesystem.py b/dlt/destinations/impl/filesystem/filesystem.py index 326fc70309..37344f8394 100644 --- a/dlt/destinations/impl/filesystem/filesystem.py +++ b/dlt/destinations/impl/filesystem/filesystem.py @@ -49,22 +49,14 @@ def __init__( fs_client, _ = fsspec_from_config(config) self.destination_file_name = make_filename(config, file_name, schema_name, load_id) - # Make sure directory exists before moving file - dir_path = self.make_remote_path(only_dir=True) - fs_client.makedirs(dir_path, exist_ok=True) - # Once directory is created we can move things item = self.make_remote_path() fs_client.put_file(local_path, item) - def make_remote_path(self, only_dir: bool = False) -> str: - if only_dir: - dir_path = FileStorage.get_dir_name_from_file_path(self.destination_file_name) - return f"{self.config.protocol}://{posixpath.join(self.dataset_path, dir_path)}" - else: - return ( - f"{self.config.protocol}://{posixpath.join(self.dataset_path, self.destination_file_name)}" - ) + def make_remote_path(self) -> str: + return ( + f"{self.config.protocol}://{posixpath.join(self.dataset_path, self.destination_file_name)}" + ) def state(self) -> TLoadJobState: return "completed"