Merge pull request #64 from uktrade/feature/celery-configuration

chore:add Celery and Django Celery Beat configuration

fbr/cache/tasks.py

@@ -8,7 +8,7 @@
 from fbr.search.utils.documents import clear_all_documents
 
 
-@shared_task()
+@shared_task(binding=True)
 def rebuild_cache():
     try:
         start = time.time()

fbr/config/celery.py

@@ -1,9 +1,20 @@
+import os
+
 from celery import Celery
+from celery.schedules import crontab
+from dbt_copilot_python.celery_health_check import healthcheck
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "fbr.config.settings.local")
 
-app = Celery("fbr_celery")
+celery_app = Celery("fbr_celery")
+celery_app.config_from_object("django.conf:settings.base", namespace="CELERY")
+celery_app.autodiscover_tasks()
 
-# Load settings from Django or directly
-app.config_from_object("django.conf:settings", namespace="CELERY")
+celery_app = healthcheck.setup(celery_app)
 
-# Auto-discover tasks in installed apps
-app.autodiscover_tasks()
+celery_app.conf.beat_schedule = {
+    "schedule-fbr-cache-task": {
+        "task": "fbr.cache.tasks.rebuild_cache",
+        "schedule": crontab(hour="1", minute="0"),
+    },
+}

fbr/config/settings/base.py

@@ -47,6 +47,7 @@
 
 # Application definition
 DJANGO_APPS = [
+    "django_celery_beat",
     "django.contrib.admin",
     "django.contrib.auth",
     "django.contrib.contenttypes",
@@ -62,7 +63,6 @@
 
 THIRD_PARTY_APPS: list = [
     "webpack_loader",
-    "django_celery_beat",
 ]
 
 INSTALLED_APPS = DJANGO_APPS + LOCAL_APPS + THIRD_PARTY_APPS
@@ -137,6 +137,18 @@
     },
 ]
 
+REDIS_ENDPOINT = env("REDIS_ENDPOINT", default="")
+
+# Celery
+CELERY_BROKER_URL = env("REDIS_ENDPOINT", default="")
+if CELERY_BROKER_URL and CELERY_BROKER_URL.startswith("rediss://"):
+    CELERY_BROKER_URL = f"{CELERY_BROKER_URL}?ssl_cert_reqs=CERT_REQUIRED"
+CELERY_RESULT_BACKEND = CELERY_BROKER_URL
+CELERY_ACCEPT_CONTENT = ["application/json"]
+CELERY_RESULT_SERIALIZER = "json"
+CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True
+CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers.DatabaseScheduler"
+
 # Internationalisation
 LANGUAGE_CODE = "en-gb"
 TIME_ZONE = "Europe/London"