Skip to content

Commit

Permalink
Apply copilot review comments
Browse files Browse the repository at this point in the history
  • Loading branch information
murdo-moj committed Dec 30, 2024
1 parent ba35f7d commit 35c33ed
Showing 1 changed file with 13 additions and 21 deletions.
34 changes: 13 additions & 21 deletions lib/datahub-client/data_platform_catalogue/client/parsers.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ def get_refresh_period_from_cadet_tags(
if schedule in tag_ref.display_name
]
if len(relevant_refresh_schedules) > 1:
logger.warn(f"More than one refresh period tag found: {tags=}")
logger.warning(f"More than one refresh period tag found: {tags=}")

if relevant_refresh_schedules:
refresh_schedule = ", ".join(relevant_refresh_schedules).capitalize()
Expand Down Expand Up @@ -483,26 +483,6 @@ def parse_metadata_last_ingested(self, entity: dict[str, Any]) -> datetime | Non
return None
return timestamp

def parse_data_last_modified(self, properties: dict[str, Any]) -> datetime | None:
"""
Return the time when the data was last updated in the source system
(not Datahub)
"""
modified = (properties.get("lastModified") or {}).get("time")
return None if modified == 0 else modified

def parse_last_datajob_run_date(self, response: dict[str, Any]) -> datetime | None:
"""
Look for the last job that produced/consumed the dataset and return the time it ran.
"""
list_of_runs: list = response.get("runs", {}).get("runs", [])
if not list_of_runs:
updated = None
if list_of_runs:
updated = list_of_runs[0].get("created", {}).get("time", {})

return updated


class DatasetParser(EntityParser):
def __init__(self):
Expand Down Expand Up @@ -613,6 +593,18 @@ def parse_to_entity_object(self, response: dict, urn: str) -> Table:
platform=EntityRef(display_name=platform_name, urn=platform_name),
)

def parse_last_datajob_run_date(self, response: dict[str, Any]) -> datetime | None:
"""
Look for the last job that produced/consumed the dataset and return the time it ran.
"""
list_of_runs: list = response.get("runs", {}).get("runs", [])
if not list_of_runs:
updated = None
if list_of_runs:
updated = list_of_runs[0].get("created", {}).get("time", {})

return updated


class TableParser(DatasetParser):
def __init__(self):
Expand Down

0 comments on commit 35c33ed

Please sign in to comment.