Merge pull request #228 from uktrade/LTD-4630-background-task-cleanup

removed background task from lite-hmrc

.circleci/config.yml

@@ -92,21 +92,9 @@ jobs:
           command: |
             pipenv run bandit -c bandit.yaml -r .
 
-  check_background_tasks_up:
-    machine:
-      image: ubuntu-2004:202010-01
-    resource_class: medium
-    working_directory: /tmp/app
-    steps:
-      - checkout
-      - run:
-          name: Run background task scheduler
-          command: docker-compose -f docker-compose-circleci-background-tasks-up.yml up --exit-code-from check-background-tasks
-
 workflows:
   version: 2
   test:
     jobs:
       - tests
       - linting
-      - check_background_tasks_up

Pipfile

@@ -20,7 +20,6 @@ watchdog = {extras = ["watchmedo"], version = "*"}
 
 [packages]
 djangorestframework = "~=3.9"
-django-background-tasks = "~=1.2"
 django-environ = "~=0.4"
 django-model-utils = "~=4.0"
 sentry-sdk = "~=1.17.0"

README.md

@@ -4,12 +4,9 @@
 This project is meant for sending licence updates to HMRC and receiving usage reporting. Information like licence updates
 and usage are exchanged as mail attachment between Lite and HMRC
 
-Tasks are managed using this project: [Django Background Tasks](https://github.com/arteria/django-background-tasks/blob/master/docs/index.rst)
-
-We currently have two mechanisms for background tasks in lite;
-- django-background-tasks: `pipenv run ./manage.py process_tasks` will run all background tasks
-- celery: a celery container is running by default when using docker-compose.  If a working copy
-    "on the metal" without docker, run celery with `watchmedo auto-restart -d . -R -p '*.py' -- celery -A conf worker -l info`
+We currently use Celery to manage tasks;
+- a celery container is running by default when using docker-compose.  
+- If a working copy "on the metal" without docker, run celery with `watchmedo auto-restart -d . -R -p '*.py' -- celery -A conf worker -l info`
 
 The entry point for configuring the tasks is defined here: `lite-hmrc/mail/apps.py`
 

codecov.yml

@@ -12,3 +12,5 @@ coverage:
         threshold: 0%
         if_ci_failed: error
         only_pulls: false
+ignore:
+  - "mock_hmrc"

conf/settings.py

@@ -36,8 +36,7 @@
     "django.contrib.messages",
     "django.contrib.staticfiles",
     "mail",
-    "background_task",
-    # This is app custome healthcheck app and health_check is django-health-check
+    # healthcheck app is for custom healthchecks in this app, health_check is django-health-check
     "healthcheck",
     "health_check",
     "health_check.db",
@@ -226,13 +225,6 @@
 # The URL used to send licence reply data to ICMS
 ICMS_API_URL = env("ICMS_API_URL", default="http://web:8080/")
 
-# Background Tasks
-BACKGROUND_TASK_ENABLED = env.bool("BACKGROUND_TASK_ENABLED", default=True)
-BACKGROUND_TASK_RUN_ASYNC = True
-# Number of times a task is retried given a failure occurs with exponential back-off = ((current_attempt ** 4) + 5)
-MAX_ATTEMPTS = 7  # e.g. 7th attempt occurs approx 40 minutes after 1st attempt (assuming instantaneous failures)
-MAX_RUN_TIME = env("BACKGROUND_TASK_MAX_RUN_TIME", default=900)
-
 # Sentry
 if env.str("SENTRY_DSN", ""):
     sentry_sdk.init(

docker.env

@@ -18,8 +18,6 @@ LITE_LICENCE_DATA_POLL_INTERVAL=1200
 EMAIL_AWAITING_REPLY_TIME=3600
 NOTIFY_USERS=["", ""]
 
-BACKGROUND_TASK_ENABLED=True
-
 HAWK_AUTHENTICATION_ENABLED=False
 LITE_HMRC_INTEGRATION_HAWK_KEY=LITE_HMRC_INTEGRATION_HAWK_KEY
 LITE_API_HAWK_KEY=LITE_API_HAWK_KEY

healthcheck/tests/test_checks.py

@@ -1,10 +1,8 @@
 import datetime
 import poplib
 import uuid
-from datetime import timedelta
 from unittest.mock import patch
 
-from background_task.models import Task
 from django.conf import settings
 from django.test import TestCase
 from django.utils import timezone
@@ -14,7 +12,6 @@
 from healthcheck.checks import LicencePayloadsHealthCheck, MailboxAuthenticationHealthCheck, PendingMailHealthCheck
 from mail.enums import LicenceActionEnum, ReceptionStatusEnum
 from mail.models import LicencePayload, Mail
-from mail.tasks import LICENCE_DATA_TASK_QUEUE
 
 
 class MailboxAuthenticationHealthCheckTest(TestCase):

local.env

@@ -17,8 +17,6 @@ LITE_LICENCE_DATA_POLL_INTERVAL=600
 EMAIL_AWAITING_REPLY_TIME=3600
 NOTIFY_USERS=["", ""]
 
-BACKGROUND_TASK_ENABLED=True
-
 HAWK_AUTHENTICATION_ENABLED=False
 LITE_HMRC_INTEGRATION_HAWK_KEY=LITE_HMRC_INTEGRATION_HAWK_KEY
 LITE_API_HAWK_KEY=LITE_API_HAWK_KEY

mail/apps.py

@@ -1,27 +1,18 @@
 from django.apps import AppConfig
-from django.conf import settings
 from django.db.models.signals import post_migrate
 
 
 class MailConfig(AppConfig):
     name = "mail"
 
     @classmethod
-    def initialize_background_tasks(cls, **kwargs):
-        from background_task.models import Task
-
+    def initialize_send_licence_usage_figures_to_lite_api(cls, **kwargs):
         from mail.celery_tasks import send_licence_usage_figures_to_lite_api
-        from mail.enums import ChiefSystemEnum
         from mail.models import UsageData
-        from mail.tasks import LICENCE_DATA_TASK_QUEUE
 
-        Task.objects.filter(queue=LICENCE_DATA_TASK_QUEUE).delete()
-        if settings.BACKGROUND_TASK_ENABLED:
-            # LITE/SPIRE Tasks
-            if settings.CHIEF_SOURCE_SYSTEM == ChiefSystemEnum.SPIRE:
-                usage_updates_not_sent_to_lite = UsageData.objects.filter(has_lite_data=True, lite_sent_at__isnull=True)
-                for obj in usage_updates_not_sent_to_lite:
-                    send_licence_usage_figures_to_lite_api.delay(str(obj.id))
+        usage_updates_not_sent_to_lite = UsageData.objects.filter(has_lite_data=True, lite_sent_at__isnull=True)
+        for obj in usage_updates_not_sent_to_lite:
+            send_licence_usage_figures_to_lite_api.delay(str(obj.id))
 
     def ready(self):
-        post_migrate.connect(self.initialize_background_tasks, sender=self)
+        post_migrate.connect(self.initialize_send_licence_usage_figures_to_lite_api, sender=self)

mail/celery_tasks.py

@@ -15,14 +15,13 @@
 from mail.enums import ReceptionStatusEnum, SourceEnum
 from mail.libraries.builders import build_licence_data_mail
 from mail.libraries.data_processors import build_request_mail_message_dto
-from mail.libraries.routing_controller import send, update_mail, check_and_route_emails
+from mail.libraries.routing_controller import check_and_route_emails, send, update_mail
 from mail.libraries.usage_data_decomposition import build_json_payload_from_data_blocks, split_edi_data_by_id
 from mail.models import LicenceIdMapping, LicencePayload, Mail, UsageData
 from mail.servers import smtp_send
 
 logger = get_task_logger(__name__)
 
-
 # Send Usage Figures to LITE API
 def get_lite_api_url():
     """The URL for the licence usage callback, from the LITE_API_URL setting.

mail/tests/test_apps.py

@@ -87,7 +87,7 @@ def test_app_initialization_processes_usage_data(self, put_request):
         )
 
         # We expect our UsageData record to be processed as part of this initialization function
-        MailConfig.initialize_background_tasks()
+        MailConfig.initialize_send_licence_usage_figures_to_lite_api()
 
         self.usage_data.refresh_from_db()
         put_request.assert_called_with(

mail/tests/test_send_licence_usage_figures_to_lite_api.py

@@ -30,7 +30,6 @@ def json(self):
         return self.json_data
 
 
-@override_settings(BACKGROUND_TASK_ENABLED=False)  # Disable task from being run on app initialization
 class UpdateUsagesTaskTests(LiteHMRCTestClient):
     def setUp(self):
         super().setUp()

mail/tests/test_send_lite_licence_updates_task.py

@@ -1,16 +1,15 @@
+from smtplib import SMTPException
 from unittest import mock
 
 from django.test import override_settings
 from parameterized import parameterized
-from smtplib import SMTPException
 
 from mail.celery_tasks import send_licence_details_to_hmrc
 from mail.enums import ReceptionStatusEnum
 from mail.models import LicencePayload, Mail
 from mail.tests.libraries.client import LiteHMRCTestClient
 
 
-@override_settings(BACKGROUND_TASK_ENABLED=False)  # Disable task from being run on app initialization
 class TaskTests(LiteHMRCTestClient):
     def setUp(self):
         super().setUp()

mock_hmrc/apps.py

@@ -6,8 +6,8 @@ class MockHmrcConfig(AppConfig):
     name = "mock_hmrc"
 
     @classmethod
-    def initialize_background_tasks(cls, **kwargs):
+    def initialize_send_licence_usage_figures_to_lite_api(cls, **kwargs):
         pass
 
     def ready(self):
-        post_migrate.connect(self.initialize_background_tasks, sender=self)
+        post_migrate.connect(self.initialize_send_licence_usage_figures_to_lite_api, sender=self)

mock_hmrc/tasks.py

@@ -1,11 +1,8 @@
 import logging
 
-from background_task import background
-
 from mock_hmrc.handler import parse_and_reply_emails
 
 
-@background(queue="handle_hmrc_replies_queue", schedule=0)
 def handle_replies():
     logging.info("Polling mock HMRC inbox for updates")
 

mock_hmrc/views.py

@@ -9,6 +9,6 @@
 
 class HandleReplies(APIView):
     def get(self, request):
-        handle_replies.now()
+        handle_replies()
 
         return HttpResponse(status=status.HTTP_200_OK)