diff --git a/common/celery.py b/common/celery.py index 1c2f8c82c..9cf54f146 100644 --- a/common/celery.py +++ b/common/celery.py @@ -20,3 +20,7 @@ # this should be automactically configured via ^^ config_from_object # but it isn't so here it's configured here app.conf.task_routes = settings.CELERY_ROUTES + +# this means the celery task only sends the acknowledgement (removes from task queue) +# after it has completed +# app.conf.task_ack_late = True # What's the impact on all of our tasks? diff --git a/notifications/tasks.py b/notifications/tasks.py index a771c71b3..78cc12d52 100644 --- a/notifications/tasks.py +++ b/notifications/tasks.py @@ -1,12 +1,20 @@ import logging from celery import shared_task +from django.conf import settings from django.db.transaction import atomic logger = logging.getLogger(__name__) -@shared_task +@shared_task( + default_retry_delay=settings.NOTIFICATIONS_DEFAULT_RETRY_DELAY, + max_retries=settings.NOTIFICATIONS_MAX_RETRIES, + retry_backoff=True, + retry_backoff_max=settings.NOTIFICATIONS_RETRY_BACKOFF_MAX, + retry_jitter=True, + autoretry_for=(Exception,), +) @atomic def send_emails_task(notification_pk: int): """Task for emailing all users signed up to receive packaging updates and diff --git a/publishing/tasks.py b/publishing/tasks.py index 57e2664f1..ea172ae89 100644 --- a/publishing/tasks.py +++ b/publishing/tasks.py @@ -11,7 +11,14 @@ logger = logging.getLogger(__name__) -@app.task +@app.task( + default_retry_delay=settings.ENVELOPE_GENERATION_DEFAULT_RETRY_DELAY, + max_retries=settings.ENVELOPE_GENERATION_MAX_RETRIES, + retry_backoff=True, + retry_backoff_max=settings.ENVELOPE_GENERATION_RETRY_BACKOFF_MAX, + retry_jitter=True, + autoretry_for=(Exception,), +) def create_xml_envelope_file( packaged_work_basket_id: int, notify_when_done: bool = True, diff --git a/publishing/tests/test_tasks.py b/publishing/tests/test_tasks.py index 4f7c0171f..7e66781b2 100644 --- a/publishing/tests/test_tasks.py +++ b/publishing/tests/test_tasks.py @@ -38,7 +38,7 @@ def test_create_and_upload_envelope( create_xml_envelope_file.apply( (packaged_work_basket.pk, True), ) - mock_save.assert_called_once() + mock_save.assert_called() assert expected_bucket in s3_bucket_names() diff --git a/settings/common.py b/settings/common.py index aa667ea4f..1eee9c599 100644 --- a/settings/common.py +++ b/settings/common.py @@ -437,16 +437,54 @@ # Settings about retrying uploads if the api cannot be contacted. # Names correspond to celery settings for retrying tasks: # https://docs.celeryq.dev/en/stable/userguide/tasks.html#automatic-retry-for-known-exceptions +CELERY_MAX_RETRIES = int( + os.environ.get("CELERY_MAX_RETRIES", "3"), +) +CELERY_RETRY_BACKOFF_MAX = int( + os.environ.get("CELERY_RETRY_BACKOFF_MAX", "600"), +) +CELERY_DEFAULT_RETRY_DELAY = int( + os.environ.get("CELERY_DEFAULT_RETRY_DELAY", "8"), +) + CROWN_DEPENDENCIES_API_MAX_RETRIES = int( - os.environ.get("CROWN_DEPENDENCIES_API_MAX_RETRIES", "3"), + os.environ.get("CROWN_DEPENDENCIES_API_MAX_RETRIES", CELERY_MAX_RETRIES), ) CROWN_DEPENDENCIES_API_RETRY_BACKOFF_MAX = int( - os.environ.get("CROWN_DEPENDENCIES_API_RETRY_BACKOFF_MAX", "600"), + os.environ.get( + "CROWN_DEPENDENCIES_API_RETRY_BACKOFF_MAX", + CELERY_RETRY_BACKOFF_MAX, + ), ) CROWN_DEPENDENCIES_API_DEFAULT_RETRY_DELAY = int( - os.environ.get("CROWN_DEPENDENCIES_API_DEFAULT_RETRY_DELAY", "8"), + os.environ.get( + "CROWN_DEPENDENCIES_API_DEFAULT_RETRY_DELAY", + CELERY_DEFAULT_RETRY_DELAY, + ), ) +ENVELOPE_GENERATION_MAX_RETRIES = int( + os.environ.get("ENVELOPE_GENERATION_MAX_RETRIES", CELERY_MAX_RETRIES), +) +ENVELOPE_GENERATION_RETRY_BACKOFF_MAX = int( + os.environ.get("ENVELOPE_GENERATION_RETRY_BACKOFF_MAX", CELERY_RETRY_BACKOFF_MAX), +) +ENVELOPE_GENERATION_DEFAULT_RETRY_DELAY = int( + os.environ.get( + "ENVELOPE_GENERATION_DEFAULT_RETRY_DELAY", + CELERY_DEFAULT_RETRY_DELAY, + ), +) + +NOTIFICATIONS_MAX_RETRIES = int( + os.environ.get("NOTIFICATIONS_MAX_RETRIES", CELERY_MAX_RETRIES), +) +NOTIFICATIONS_RETRY_BACKOFF_MAX = int( + os.environ.get("NOTIFICATIONS_RETRY_BACKOFF_MAX", CELERY_RETRY_BACKOFF_MAX), +) +NOTIFICATIONS_DEFAULT_RETRY_DELAY = int( + os.environ.get("NOTIFICATIONS_DEFAULT_RETRY_DELAY", CELERY_DEFAULT_RETRY_DELAY), +) # SQLite AWS settings SQLITE_STORAGE_BUCKET_NAME = os.environ.get("SQLITE_STORAGE_BUCKET_NAME", "sqlite")