Merge pull request #216 from uktrade/LTD-4585-add-celery

Ltd 4585 add celery

.gitignore

@@ -78,3 +78,4 @@ mailserver/*
 
 # pii file
 .pii-secret-hook
+celerybeat-schedule

.pre-commit-config.yaml

@@ -6,17 +6,17 @@ repos:
           - id: black
           # Config for black lives in pyproject.toml
     - repo: https://github.com/asottile/blacken-docs
-      rev: v1.12.1
+      rev: 1.16.0
       hooks:
           - id: blacken-docs
             additional_dependencies: [black==22.3.0]
     - repo: https://github.com/pycqa/isort
-      rev: 5.10.1
+      rev: 5.13.2
       hooks:
           - id: isort
             name: isort (python)
     - repo: https://github.com/pre-commit/pre-commit-hooks
-      rev: v4.1.0
+      rev: v4.5.0
       hooks:
           - id: trailing-whitespace
             args: ["--markdown-linebreak-ext=md,markdown"]
@@ -30,7 +30,7 @@ repos:
           - id: detect-aws-credentials
             args: ["--allow-missing-credentials"]
     - repo: https://github.com/uktrade/pii-secret-check-hooks
-      rev: 0.0.0.28
+      rev: 0.0.0.35
       hooks:
           - id: pii_secret_filename
             files: ''

Dockerfile

@@ -6,10 +6,13 @@ RUN apt update --fix-missing  \
     && apt install libpq-dev gcc -y  \
     && apt install build-essential -y --no-install-recommends
 
-RUN pip3 install pipenv
+ENV HOME /root
+ENV PYENV_ROOT $HOME/.pyenv
+ENV PATH $PYENV_ROOT/bin:$PATH
 
+RUN pip3 install pipenv
 ADD Pipfile* /app/
+RUN pipenv install --dev --deploy
 
-RUN pipenv sync --dev
 
 ADD . /app

Pipfile

@@ -16,6 +16,7 @@ pytest = "*"
 pytest-django = "*"
 pytest-cov = "*"
 requests-mock = "*"
+watchdog = {extras = ["watchmedo"], version = "*"}
 
 [packages]
 djangorestframework = "~=3.9"
@@ -36,6 +37,8 @@ django-jsonfield = "==1.4"
 msal = "~=1.22.0"
 psycopg2-binary = "~=2.9.3"
 setuptools = "~=65.5.1"
+celery = "~=5.2.7"
+redis = "~=4.0.2"
 
 [requires]
 python_version = "3.8"

Procfile

@@ -1,3 +1,5 @@
 # See gunicorn.conf.py for more configuration.
 web: python manage.py migrate && gunicorn conf.wsgi:application
 worker: python manage.py process_tasks --log-std
+celeryworker: celery -A conf worker -l info
+celerybeat: celery -A conf beat -l info

README.md

@@ -6,6 +6,11 @@ and usage are exchanged as mail attachment between Lite and HMRC
 
 Tasks are managed using this project: [Django Background Tasks](https://github.com/arteria/django-background-tasks/blob/master/docs/index.rst)
 
+We currently have two mechanisms for background tasks in lite;
+- django-background-tasks: `pipenv run ./manage.py process_tasks` will run all background tasks
+- celery: a celery container is running by default when using docker-compose.  If a working copy
+    "on the metal" without docker, run celery with `watchmedo auto-restart -d . -R -p '*.py' -- celery -A conf worker -l info`
+
 The entry point for configuring the tasks is defined here: `lite-hmrc/mail/apps.py`
 
 

conf/__init__.py

@@ -0,0 +1,3 @@
+from .celery import app as celery_app
+
+__all__ = ("celery_app",)

conf/celery.py

@@ -0,0 +1,17 @@
+import os
+
+from celery import Celery
+
+# Set the default Django settings module for the 'celery' program.
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "conf.settings")
+
+app = Celery("hmrc")
+
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object("django.conf:settings", namespace="CELERY")
+
+# Load celery_tasks.py modules from all registered Django apps.
+app.autodiscover_tasks(related_name="celery_tasks")

conf/settings.py

@@ -1,6 +1,7 @@
 import os
 import sys
 import uuid
+from urllib.parse import urlencode
 
 import sentry_sdk
 from django_log_formatter_ecs import ECSFormatter
@@ -25,7 +26,6 @@
 
 ALLOWED_HOSTS = "*"
 
-
 # Application definition
 
 INSTALLED_APPS = [
@@ -255,3 +255,31 @@
 AZURE_AUTH_TENANT_ID = env.str("AZURE_AUTH_TENANT_ID")
 
 SEND_REJECTED_EMAIL = env.bool("SEND_REJECTED_EMAIL", default=True)
+
+DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
+
+VCAP_SERVICES = env.json("VCAP_SERVICES", {})
+
+if "redis" in VCAP_SERVICES:
+    REDIS_BASE_URL = VCAP_SERVICES["redis"][0]["credentials"]["uri"]
+else:
+    REDIS_BASE_URL = env("REDIS_BASE_URL", default=None)
+
+
+def _build_redis_url(base_url, db_number, **query_args):
+    encoded_query_args = urlencode(query_args)
+    return f"{base_url}/{db_number}?{encoded_query_args}"
+
+
+if REDIS_BASE_URL:
+    # Give celery tasks their own redis DB - future uses of redis should use a different DB
+    REDIS_CELERY_DB = env("REDIS_CELERY_DB", default=0)
+    is_redis_ssl = REDIS_BASE_URL.startswith("rediss://")
+    url_args = {"ssl_cert_reqs": "CERT_REQUIRED"} if is_redis_ssl else {}
+
+    CELERY_BROKER_URL = _build_redis_url(REDIS_BASE_URL, REDIS_CELERY_DB, **url_args)
+    CELERY_RESULT_BACKEND = CELERY_BROKER_URL
+
+CELERY_TASK_ALWAYS_EAGER = env.bool("CELERY_TASK_ALWAYS_EAGER", False)
+CELERY_TASK_STORE_EAGER_RESULT = env.bool("CELERY_TASK_STORE_EAGER_RESULT", False)
+CELERY_TASK_SEND_SENT_EVENT = env.bool("CELERY_TASK_SEND_SENT_EVENT", True)

core/celery_tasks.py

@@ -0,0 +1,27 @@
+from celery import shared_task
+
+from mail.models import Mail
+
+
+@shared_task
+def debug_add(x, y):
+    """
+    Simple debug celery task to add two numbers.
+    """
+    return x + y
+
+
+@shared_task
+def debug_count_mail():
+    """
+    Simple debug celery task to count the number of mail in the app.
+    """
+    return Mail.objects.count()
+
+
+@shared_task
+def debug_exception():
+    """
+    Debug task which raises an exception.
+    """
+    raise Exception("debug_exception task")

core/tests/test_celery_tasks.py

@@ -0,0 +1,29 @@
+from datetime import timedelta
+
+from django.conf import settings
+from django.utils import timezone
+from rest_framework.test import APITestCase
+
+from core import celery_tasks
+from mail.enums import ReplyStatusEnum
+from mail.models import Mail
+
+
+class CeleryMailTest(APITestCase):
+    def test_debug_add(self):
+        res = celery_tasks.debug_add(1, 2)
+        assert res == 3
+
+    def test_debug_exception(self):
+        self.assertRaises(Exception, celery_tasks.debug_exception)
+
+    def test_debug_count_mail(self):
+        sent_at = timezone.now() - timedelta(seconds=settings.EMAIL_AWAITING_REPLY_TIME)
+        Mail.objects.create(
+            edi_filename="filename",
+            edi_data="1\\fileHeader\\CHIEF\\SPIRE\\",
+            status=ReplyStatusEnum.PENDING,
+            sent_at=sent_at,
+        )
+        res = celery_tasks.debug_count_mail()
+        assert res == 1

docker-compose.yml

@@ -27,6 +27,8 @@ services:
       - 8000:8000
     depends_on:
       - lite-hmrc-postgres
+      - celery
+      - celery-beat
     expose:
       - 8000
     command: pipenv run ./manage.py runserver 0.0.0.0:8000
@@ -37,5 +39,32 @@ services:
       - 587:1025  # SMTP
     image: mailhog/mailhog
 
+  celery:
+    build: .
+    volumes:
+      - .:/app
+    env_file: .env
+    links:
+      - lite-hmrc-postgres
+      - redis
+    command: pipenv run watchmedo auto-restart -d . -R -p '*.py' -- celery -A conf worker -l info
+
+  celery-beat:
+    build: .
+    volumes:
+      - .:/app
+    env_file: .env
+    links:
+      - lite-hmrc-postgres
+      - redis
+    command:  pipenv run watchmedo auto-restart -d . -R -p '*.py' -- celery -A conf beat
+
+  redis:
+    image: "redis:5-alpine"
+    container_name: hmrc-redis
+    expose:
+      - 6379
+    ports:
+      - 6379:6379
 volumes:
   maildata:

docker.env

@@ -59,3 +59,4 @@ LITE_API_URL=https://lite.example.com
 AZURE_AUTH_CLIENT_ID=
 AZURE_AUTH_CLIENT_SECRET=
 AZURE_AUTH_TENANT_ID=
+REDIS_BASE_URL=redis://redis:6379

local.env

@@ -58,3 +58,4 @@ AZURE_AUTH_CLIENT_SECRET=<FROM_VAULT>
 AZURE_AUTH_TENANT_ID=<FROM_VAULT>
 
 SEND_REJECTED_EMAIL=True
+REDIS_BASE_URL=redis://localhost:6379

mail/icms_serializers.py

@@ -36,7 +36,6 @@ class AddressSerializer(serializers.Serializer):
 
 
 class OrganisationSerializer(serializers.Serializer):
-
     # "GB" + 12 or 15 digits.
     eori_number = serializers.CharField(min_length=14, max_length=17)
     name = serializers.CharField(max_length=80)

mail/libraries/chieftypes.py

@@ -4,6 +4,7 @@
 # These types represent the structure of records in a CHIEF message. A message
 # is one of licenceData, licenceReply, usageData, or usageReply types.
 
+
 # Every line in a message starts with a line number, and then the record type.
 @dataclass
 class _Record:

mail/libraries/usage_data_decomposition.py

@@ -106,7 +106,6 @@ def build_json_payload_from_data_blocks(data_blocks: list) -> dict:
                 licence_payload["id"] = get_licence_id(licence_reference)
 
             if "line" == line_array[0]:
-
                 good_payload["id"] = get_good_id(line_number=line_array[1], licence_reference=licence_reference)
                 good_payload["usage"] = line_array[2]
                 good_payload["value"] = line_array[3]

mail/tests/test_set_skip_flag_on_payload.py

@@ -25,7 +25,6 @@ def get_payload(self, **values):
         ]
     )
     def test_default_skip_process(self, skip_process, expected):
-
         payload = self.get_payload()
         call_command("set_skip_flag_on_payload", "--reference", payload.reference, "--skip_process", skip_process)
         payload.refresh_from_db()
@@ -38,7 +37,6 @@ def test_default_skip_process(self, skip_process, expected):
         ]
     )
     def test_default_skip_process_dry_run(self, skip_process, expected):
-
         payload = self.get_payload()
         call_command(
             "set_skip_flag_on_payload",
@@ -58,7 +56,6 @@ def test_default_skip_process_dry_run(self, skip_process, expected):
         ]
     )
     def test_default_skip_process_processed_true(self, skip_process, expected):
-
         payload = self.get_payload()
         payload.is_processed = True
         payload.save()