diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/macros.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/macros.py index 8b8c007a8c5e..9fb5fe99a5a6 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/macros.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/macros.py @@ -4,9 +4,11 @@ import builtins import datetime -import numbers +import typing from typing import Union +import isodate +import pytz from dateutil import parser from isodate import parse_duration @@ -15,7 +17,7 @@ """ -def now_utc(): +def now_utc() -> datetime.datetime: """ Current local date and time in UTC timezone @@ -25,7 +27,7 @@ def now_utc(): return datetime.datetime.now(datetime.timezone.utc) -def today_utc(): +def today_utc() -> datetime.date: """ Current date in UTC timezone @@ -35,7 +37,7 @@ def today_utc(): return datetime.datetime.now(datetime.timezone.utc).date() -def timestamp(dt: Union[numbers.Number, str]): +def timestamp(dt: Union[float, str]) -> Union[int, float]: """ Converts a number or a string to a timestamp @@ -48,21 +50,21 @@ def timestamp(dt: Union[numbers.Number, str]): :param dt: datetime to convert to timestamp :return: unix timestamp """ - if isinstance(dt, numbers.Number): + if isinstance(dt, (int, float)): return int(dt) else: - return _str_to_datetime(dt).astimezone(datetime.timezone.utc).timestamp() + return _str_to_datetime(dt).astimezone(pytz.utc).timestamp() def _str_to_datetime(s: str) -> datetime.datetime: parsed_date = parser.isoparse(s) if not parsed_date.tzinfo: # Assume UTC if the input does not contain a timezone - parsed_date = parsed_date.replace(tzinfo=datetime.timezone.utc) - return parsed_date.astimezone(datetime.timezone.utc) + parsed_date = parsed_date.replace(tzinfo=pytz.utc) + return parsed_date.astimezone(pytz.utc) -def max(*args): +def max(*args: typing.Any) -> typing.Any: """ Returns biggest object of an iterable, or two or more arguments. @@ -95,7 +97,7 @@ def day_delta(num_days: int, format: str = "%Y-%m-%dT%H:%M:%S.%f%z") -> str: return (datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=num_days)).strftime(format) -def duration(datestring: str) -> datetime.timedelta: +def duration(datestring: str) -> Union[datetime.timedelta, isodate.Duration]: """ Converts ISO8601 duration to datetime.timedelta @@ -111,10 +113,17 @@ def format_datetime(dt: Union[str, datetime.datetime], format: str) -> str: Usage: `"{{ format_datetime(config.start_date, '%Y-%m-%d') }}"` + + CPython Datetime package has known bug with `stfrtime` method: '%s' formatting uses locale timezone + https://github.com/python/cpython/issues/77169 + https://github.com/python/cpython/issues/56959 """ if isinstance(dt, datetime.datetime): return dt.strftime(format) - return _str_to_datetime(dt).strftime(format) + dt_datetime = _str_to_datetime(dt) + if format == "%s": + return str(int(dt_datetime.timestamp())) + return dt_datetime.strftime(format) _macros_list = [now_utc, today_utc, timestamp, max, day_delta, duration, format_datetime] diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_macros.py b/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_macros.py index bfd1fbc137d0..1b9bd61cec4f 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_macros.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_macros.py @@ -71,3 +71,10 @@ def test_timestamp(test_name, input_value, expected_output): timestamp_function = macros["timestamp"] actual_output = timestamp_function(input_value) assert actual_output == expected_output + + +def test_utc_datetime_to_local_timestamp_conversion(): + """ + This test ensures correct timezone handling independent of the timezone of the system on which the sync is running. + """ + assert macros["format_datetime"](dt="2020-10-01T00:00:00Z", format="%s") == "1601510400"