diff --git a/target_s3/formats/format_base.py b/target_s3/formats/format_base.py index 216232a..ed10062 100644 --- a/target_s3/formats/format_base.py +++ b/target_s3/formats/format_base.py @@ -117,23 +117,55 @@ def create_key(self) -> str: grain = DATE_GRAIN[self.config["append_date_to_prefix_grain"].lower()] partition_name_enabled = False if self.config["partition_name_enabled"]: - partition_name_enabled = self.config["partition_name_enabled"] - folder_path += self.create_folder_structure(batch_start, grain, partition_name_enabled) + partition_name_enabled = self.config["partition_name_enabled"] + folder_path += self.create_folder_structure( + batch_start, grain, partition_name_enabled + ) if self.config["append_date_to_filename"]: grain = DATE_GRAIN[self.config["append_date_to_filename_grain"].lower()] file_name += f"{self.create_file_structure(batch_start, grain)}" return f"{folder_path}{file_name}" - def create_folder_structure(self, batch_start: datetime, grain: int, partition_name_enabled: bool) -> str: + def create_folder_structure( + self, batch_start: datetime, grain: int, partition_name_enabled: bool + ) -> str: ret = "" - ret += f"{'year=' if partition_name_enabled else ''}{batch_start.year}/" if grain <= DATE_GRAIN["year"] else "" - ret += f"{'month=' if partition_name_enabled else ''}{batch_start.month:02}/" if grain <= DATE_GRAIN["month"] else "" - ret += f"{'day=' if partition_name_enabled else ''}{batch_start.day:02}/" if grain <= DATE_GRAIN["day"] else "" - ret += f"{'hour=' if partition_name_enabled else ''}{batch_start.hour:02}/" if grain <= DATE_GRAIN["hour"] else "" - ret += f"{'minute=' if partition_name_enabled else ''}{batch_start.minute:02}/" if grain <= DATE_GRAIN["minute"] else "" - ret += f"{'second=' if partition_name_enabled else ''}{batch_start.second:02}/" if grain <= DATE_GRAIN["second"] else "" - ret += f"{'microsecond=' if partition_name_enabled else ''}{batch_start.microsecond}/" if grain <= DATE_GRAIN["microsecond"] else "" + ret += ( + f"{'year=' if partition_name_enabled else ''}{batch_start.year}/" + if grain <= DATE_GRAIN["year"] + else "" + ) + ret += ( + f"{'month=' if partition_name_enabled else ''}{batch_start.month:02}/" + if grain <= DATE_GRAIN["month"] + else "" + ) + ret += ( + f"{'day=' if partition_name_enabled else ''}{batch_start.day:02}/" + if grain <= DATE_GRAIN["day"] + else "" + ) + ret += ( + f"{'hour=' if partition_name_enabled else ''}{batch_start.hour:02}/" + if grain <= DATE_GRAIN["hour"] + else "" + ) + ret += ( + f"{'minute=' if partition_name_enabled else ''}{batch_start.minute:02}/" + if grain <= DATE_GRAIN["minute"] + else "" + ) + ret += ( + f"{'second=' if partition_name_enabled else ''}{batch_start.second:02}/" + if grain <= DATE_GRAIN["second"] + else "" + ) + ret += ( + f"{'microsecond=' if partition_name_enabled else ''}{batch_start.microsecond}/" + if grain <= DATE_GRAIN["microsecond"] + else "" + ) return ret def create_file_structure(self, batch_start: datetime, grain: int) -> str: @@ -144,7 +176,9 @@ def create_file_structure(self, batch_start: datetime, grain: int) -> str: ret += f"-{batch_start.hour:02}" if grain <= DATE_GRAIN["hour"] else "" ret += f"{batch_start.minute:02}" if grain <= DATE_GRAIN["minute"] else "" ret += f"{batch_start.second:02}" if grain <= DATE_GRAIN["second"] else "" - ret += f"{batch_start.microsecond}" if grain <= DATE_GRAIN["microsecond"] else "" + ret += ( + f"{batch_start.microsecond}" if grain <= DATE_GRAIN["microsecond"] else "" + ) return ret def flatten_key(self, k, parent_key, sep) -> str: diff --git a/target_s3/target.py b/target_s3/target.py index 6b6e2c1..99c08c4 100644 --- a/target_s3/target.py +++ b/target_s3/target.py @@ -1,6 +1,8 @@ """s3 target class.""" from __future__ import annotations +import decimal +import json from singer_sdk.target_base import Target from singer_sdk import typing as th @@ -47,7 +49,7 @@ class Targets3(Target): resulting parquet file based on taps. Doesn't \ work with 'anyOf' types or when complex data is\ not defined at element level. Doesn't work with \ - validate option for now." + validate option for now.", ), ), required=False,