Skip to content

Commit

Permalink
fix: gracefully shutdown on crash (#179)
Browse files Browse the repository at this point in the history
  • Loading branch information
polomarcus authored May 29, 2024
1 parent ced1859 commit c0bbb71
Showing 1 changed file with 21 additions and 17 deletions.
38 changes: 21 additions & 17 deletions quotaclimat/data_processing/mediatree/api_import.py
Original file line number Diff line number Diff line change
Expand Up @@ -244,23 +244,27 @@ def log_dataframe_size(df, channel):

async def main():
with monitor(monitor_slug='mediatree'): #https://docs.sentry.io/platforms/python/crons/
logging.info("Start api mediatree import")
create_tables()

event_finish = asyncio.Event()
# Start the health check server in the background
health_check_task = asyncio.create_task(run_health_check_server())

# Start batch job
if(os.environ.get("UPDATE") == "true"):
asyncio.create_task(update_pg_data(event_finish))
else:
asyncio.create_task(get_and_save_api_data(event_finish))

# Wait for both tasks to complete
await event_finish.wait()

res=health_check_task.cancel()
try:
logging.info("Start api mediatree import")
create_tables()

event_finish = asyncio.Event()
# Start the health check server in the background
health_check_task = asyncio.create_task(run_health_check_server())

# Start batch job
if(os.environ.get("UPDATE") == "true"):
asyncio.create_task(update_pg_data(event_finish))
else:
asyncio.create_task(get_and_save_api_data(event_finish))

# Wait for both tasks to complete
await event_finish.wait()

res=health_check_task.cancel()
except Exception as err:
logging.fatal("Main crash (%s) %s" % (type(err).__name__, err))
sys.exit(1)
logging.info("Exiting with success")
sys.exit(0)

Expand Down

1 comment on commit c0bbb71

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Coverage

Coverage Report
FileStmtsMissCoverMissing
postgres
   insert_data.py44784%36–38, 57–59, 64
   insert_existing_data_example.py19384%25–27
postgres/schemas
   models.py1421093%116–123, 135–136, 194–195, 209–210
quotaclimat/data_ingestion
   scrap_sitemap.py1341787%27–28, 33–34, 66–71, 95–97, 138–140, 202, 223–228
quotaclimat/data_ingestion/ingest_db
   ingest_sitemap_in_db.py553733%21–42, 45–58, 62–73
quotaclimat/data_ingestion/scrap_html
   scrap_description_article.py36392%19–20, 32
quotaclimat/data_processing/mediatree
   api_import.py18811439%42–46, 51–63, 67–70, 76, 79–112, 118–133, 137–138, 151–163, 167–173, 186–197, 200–204, 210, 237–238, 242, 246–269, 272–274
   channel_program.py91990%21–23, 34–36, 50, 86, 95
   config.py15287%7, 16
   detect_keywords.py180498%178, 230–232
   update_pg_keywords.py443032%14–84, 105–106, 127–152, 158
   utils.py662365%18, 29–53, 56, 65, 81–82
quotaclimat/utils
   healthcheck_config.py291452%22–24, 27–38
   logger.py241154%22–24, 28–37
   sentry.py10280%21–22
TOTAL110328674% 

Tests Skipped Failures Errors Time
79 0 💤 0 ❌ 0 🔥 56.971s ⏱️

Please sign in to comment.