Skip to content

Commit

Permalink
Maxi297/fix datetime format inference issue (#30442)
Browse files Browse the repository at this point in the history
  • Loading branch information
maxi297 authored Sep 15, 2023
1 parent d71facc commit 3e41ce7
Show file tree
Hide file tree
Showing 2 changed files with 9 additions and 4 deletions.
12 changes: 8 additions & 4 deletions airbyte-cdk/python/airbyte_cdk/utils/datetime_format_inferrer.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,14 @@ def _can_be_datetime(self, value: Any) -> bool:
This is the case if the value is a string or an integer between 1_000_000_000 and 2_000_000_000 for seconds
or between 1_000_000_000_000 and 2_000_000_000_000 for milliseconds.
This is separate from the format check for performance reasons"""
for timestamp_range in self._timestamp_heuristic_ranges:
if isinstance(value, str) and (not value.isdecimal() or int(value) in timestamp_range):
return True
if isinstance(value, int) and value in timestamp_range:
if isinstance(value, (str, int)):
try:
value_as_int = int(value)
for timestamp_range in self._timestamp_heuristic_ranges:
if value_as_int in timestamp_range:
return True
except ValueError:
# given that it's not parsable as an int, it can represent a datetime with one of the self._formats
return True
return False

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
("timestamp_ms_match_string", [{"d": "1686058051000"}], {"d": "%ms"}),
("timestamp_no_match_integer", [{"d": 99}], {}),
("timestamp_no_match_string", [{"d": "99999999999999999999"}], {}),
("timestamp_overflow", [{"d": f"{10**100}_100"}], {}), # this case was previously causing OverflowError hence this test
("simple_no_match", [{"d": "20220203"}], {}),
("multiple_match", [{"d": "2022-02-03", "e": "2022-02-03"}], {"d": "%Y-%m-%d", "e": "%Y-%m-%d"}),
(
Expand Down

0 comments on commit 3e41ce7

Please sign in to comment.