diff --git a/.circleci/config.yml b/.circleci/config.yml index 090cf3ae..0e4cb89a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -92,21 +92,9 @@ jobs: command: | pipenv run bandit -c bandit.yaml -r . - check_background_tasks_up: - machine: - image: ubuntu-2004:202010-01 - resource_class: medium - working_directory: /tmp/app - steps: - - checkout - - run: - name: Run background task scheduler - command: docker-compose -f docker-compose-circleci-background-tasks-up.yml up --exit-code-from check-background-tasks - workflows: version: 2 test: jobs: - tests - linting - - check_background_tasks_up diff --git a/Pipfile b/Pipfile index 1b921e38..a6ca945a 100644 --- a/Pipfile +++ b/Pipfile @@ -20,7 +20,6 @@ watchdog = {extras = ["watchmedo"], version = "*"} [packages] djangorestframework = "~=3.9" -django-background-tasks = "~=1.2" django-environ = "~=0.4" django-model-utils = "~=4.0" sentry-sdk = "~=1.17.0" diff --git a/Pipfile.lock b/Pipfile.lock index 03568937..caa52f8e 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "af7e0daa6550ce579f7c5681a7679ce9a0ef1e16d8eb41540fdbfd1ea9252599" + "sha256": "676dd01f9d7f01b4e1a1a2735334f2e65705880df342f68b33bc2d1a4ccaaaf2" }, "pipfile-spec": 6, "requires": { @@ -322,19 +322,6 @@ "markers": "python_version >= '3.6'", "version": "==3.2.23" }, - "django-background-tasks": { - "hashes": [ - "sha256:e1b19e8d495a276c9d64c5a1ff8b41132f75d2f58e45be71b78650dad59af9de" - ], - "index": "pypi", - "version": "==1.2.5" - }, - "django-compat": { - "hashes": [ - "sha256:3ac9a3bedc56b9365d9eb241bc5157d0c193769bf995f9a78dc1bc24e7c2331b" - ], - "version": "==1.0.15" - }, "django-environ": { "hashes": [ "sha256:0ff95ab4344bfeff693836aa978e6840abef2e2f1145adff7735892711590c05", @@ -1431,12 +1418,12 @@ }, "pytest-django": { "hashes": [ - "sha256:4e1c79d5261ade2dd58d91208017cd8f62cb4710b56e012ecd361d15d5d662a2", - "sha256:92d6fd46b1d79b54fb6b060bbb39428073396cec717d5f2e122a990d4b6aa5e8" + "sha256:5d054fe011c56f3b10f978f41a8efb2e5adfc7e680ef36fb571ada1f24779d90", + "sha256:ca1ddd1e0e4c227cf9e3e40a6afc6d106b3e70868fd2ac5798a22501271cd0c7" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==4.7.0" + "version": "==4.8.0" }, "pytz": { "hashes": [ diff --git a/README.md b/README.md index c46aa7ec..7cc1bbc1 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,9 @@ This project is meant for sending licence updates to HMRC and receiving usage reporting. Information like licence updates and usage are exchanged as mail attachment between Lite and HMRC -Tasks are managed using this project: [Django Background Tasks](https://github.com/arteria/django-background-tasks/blob/master/docs/index.rst) - -We currently have two mechanisms for background tasks in lite; -- django-background-tasks: `pipenv run ./manage.py process_tasks` will run all background tasks -- celery: a celery container is running by default when using docker-compose. If a working copy - "on the metal" without docker, run celery with `watchmedo auto-restart -d . -R -p '*.py' -- celery -A conf worker -l info` +We currently use Celery to manage tasks; +- a celery container is running by default when using docker-compose. +- If a working copy "on the metal" without docker, run celery with `watchmedo auto-restart -d . -R -p '*.py' -- celery -A conf worker -l info` The entry point for configuring the tasks is defined here: `lite-hmrc/mail/apps.py` diff --git a/background_tasks_up.sh b/background_tasks_up.sh deleted file mode 100755 index 3fa1cdbc..00000000 --- a/background_tasks_up.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -pipenv run ./manage.py migrate -pipenv run ./manage.py emit_test_background_task -pipenv run ./manage.py process_tasks --queue test_queue diff --git a/codecov.yml b/codecov.yml index 33267f95..aa3193f6 100644 --- a/codecov.yml +++ b/codecov.yml @@ -12,3 +12,5 @@ coverage: threshold: 0% if_ci_failed: error only_pulls: false +ignore: + - "mock_hmrc" diff --git a/conf/settings.py b/conf/settings.py index aad67315..9234ede2 100644 --- a/conf/settings.py +++ b/conf/settings.py @@ -36,8 +36,7 @@ "django.contrib.messages", "django.contrib.staticfiles", "mail", - "background_task", - # This is app custome healthcheck app and health_check is django-health-check + # healthcheck app is for custom healthchecks in this app, health_check is django-health-check "healthcheck", "health_check", "health_check.db", @@ -226,13 +225,6 @@ # The URL used to send licence reply data to ICMS ICMS_API_URL = env("ICMS_API_URL", default="http://web:8080/") -# Background Tasks -BACKGROUND_TASK_ENABLED = env.bool("BACKGROUND_TASK_ENABLED", default=True) -BACKGROUND_TASK_RUN_ASYNC = True -# Number of times a task is retried given a failure occurs with exponential back-off = ((current_attempt ** 4) + 5) -MAX_ATTEMPTS = 7 # e.g. 7th attempt occurs approx 40 minutes after 1st attempt (assuming instantaneous failures) -MAX_RUN_TIME = env("BACKGROUND_TASK_MAX_RUN_TIME", default=900) - # Sentry if env.str("SENTRY_DSN", ""): sentry_sdk.init( diff --git a/docker-compose-circleci-background-tasks-up.yml b/docker-compose-circleci-background-tasks-up.yml deleted file mode 100644 index b94b9d0d..00000000 --- a/docker-compose-circleci-background-tasks-up.yml +++ /dev/null @@ -1,49 +0,0 @@ -version: '3' - -services: - lite-hmrc-postgres: - container_name: "lite-hmrc-postgres" - environment: - - LITEHMRC_DB_USER=postgres - - LITEHMRC_DB_PASSWORD=password - - LITEHMRC_DB_SCHEMA=postgres - - POSTGRES_PASSWORD=password - build: './database' - expose: - - 5432 - ports: - - 5432:5432 - - lite-hmrc-intg: - container_name: "lite-hmrc-intg" - build: - context: . - dockerfile: ./circleci.Dockerfile - env_file: - - local.env - environment: - - DATABASE_URL=postgres://postgres:password@lite-hmrc-postgres:5432/postgres - volumes: - - .:/app - ports: - - 8000:8000 - depends_on: - - lite-hmrc-postgres - expose: - - 8000 - entrypoint: dockerize -wait tcp://lite-hmrc-postgres:5432 -timeout 300s - command: ./background_tasks_up.sh - - check-background-tasks: - image: jwilder/dockerize - entrypoint: dockerize -wait file:///app/.background-tasks-is-ready -timeout 300s - env_file: - - local.env - environment: - - DATABASE_URL=postgres://postgres:password@lite-hmrc-postgres:5432/postgres - volumes: - - .:/app - command: | - true - depends_on: - - lite-hmrc-intg diff --git a/docker.env b/docker.env index e536dad9..ebcee82c 100644 --- a/docker.env +++ b/docker.env @@ -18,8 +18,6 @@ LITE_LICENCE_DATA_POLL_INTERVAL=1200 EMAIL_AWAITING_REPLY_TIME=3600 NOTIFY_USERS=["", ""] -BACKGROUND_TASK_ENABLED=True - HAWK_AUTHENTICATION_ENABLED=False LITE_HMRC_INTEGRATION_HAWK_KEY=LITE_HMRC_INTEGRATION_HAWK_KEY LITE_API_HAWK_KEY=LITE_API_HAWK_KEY diff --git a/healthcheck/tests/test_checks.py b/healthcheck/tests/test_checks.py index 08787a83..65d98e3c 100644 --- a/healthcheck/tests/test_checks.py +++ b/healthcheck/tests/test_checks.py @@ -1,10 +1,8 @@ import datetime import poplib import uuid -from datetime import timedelta from unittest.mock import patch -from background_task.models import Task from django.conf import settings from django.test import TestCase from django.utils import timezone @@ -14,7 +12,6 @@ from healthcheck.checks import LicencePayloadsHealthCheck, MailboxAuthenticationHealthCheck, PendingMailHealthCheck from mail.enums import LicenceActionEnum, ReceptionStatusEnum from mail.models import LicencePayload, Mail -from mail.tasks import LICENCE_DATA_TASK_QUEUE class MailboxAuthenticationHealthCheckTest(TestCase): diff --git a/local.env b/local.env index e6c9be22..b810ca4f 100644 --- a/local.env +++ b/local.env @@ -17,8 +17,6 @@ LITE_LICENCE_DATA_POLL_INTERVAL=600 EMAIL_AWAITING_REPLY_TIME=3600 NOTIFY_USERS=["", ""] -BACKGROUND_TASK_ENABLED=True - HAWK_AUTHENTICATION_ENABLED=False LITE_HMRC_INTEGRATION_HAWK_KEY=LITE_HMRC_INTEGRATION_HAWK_KEY LITE_API_HAWK_KEY=LITE_API_HAWK_KEY diff --git a/mail/apps.py b/mail/apps.py index 73c1fa60..0f821b51 100644 --- a/mail/apps.py +++ b/mail/apps.py @@ -1,5 +1,4 @@ from django.apps import AppConfig -from django.conf import settings from django.db.models.signals import post_migrate @@ -7,21 +6,13 @@ class MailConfig(AppConfig): name = "mail" @classmethod - def initialize_background_tasks(cls, **kwargs): - from background_task.models import Task - + def initialize_send_licence_usage_figures_to_lite_api(cls, **kwargs): from mail.celery_tasks import send_licence_usage_figures_to_lite_api - from mail.enums import ChiefSystemEnum from mail.models import UsageData - from mail.tasks import LICENCE_DATA_TASK_QUEUE - Task.objects.filter(queue=LICENCE_DATA_TASK_QUEUE).delete() - if settings.BACKGROUND_TASK_ENABLED: - # LITE/SPIRE Tasks - if settings.CHIEF_SOURCE_SYSTEM == ChiefSystemEnum.SPIRE: - usage_updates_not_sent_to_lite = UsageData.objects.filter(has_lite_data=True, lite_sent_at__isnull=True) - for obj in usage_updates_not_sent_to_lite: - send_licence_usage_figures_to_lite_api.delay(str(obj.id)) + usage_updates_not_sent_to_lite = UsageData.objects.filter(has_lite_data=True, lite_sent_at__isnull=True) + for obj in usage_updates_not_sent_to_lite: + send_licence_usage_figures_to_lite_api.delay(str(obj.id)) def ready(self): - post_migrate.connect(self.initialize_background_tasks, sender=self) + post_migrate.connect(self.initialize_send_licence_usage_figures_to_lite_api, sender=self) diff --git a/mail/celery_tasks.py b/mail/celery_tasks.py index f4b6db22..3d82d571 100644 --- a/mail/celery_tasks.py +++ b/mail/celery_tasks.py @@ -15,14 +15,13 @@ from mail.enums import ReceptionStatusEnum, SourceEnum from mail.libraries.builders import build_licence_data_mail from mail.libraries.data_processors import build_request_mail_message_dto -from mail.libraries.routing_controller import send, update_mail, check_and_route_emails +from mail.libraries.routing_controller import check_and_route_emails, send, update_mail from mail.libraries.usage_data_decomposition import build_json_payload_from_data_blocks, split_edi_data_by_id from mail.models import LicenceIdMapping, LicencePayload, Mail, UsageData from mail.servers import smtp_send logger = get_task_logger(__name__) - # Send Usage Figures to LITE API def get_lite_api_url(): """The URL for the licence usage callback, from the LITE_API_URL setting. diff --git a/mail/management/commands/emit_test_background_task.py b/mail/management/commands/emit_test_background_task.py deleted file mode 100644 index fc825c61..00000000 --- a/mail/management/commands/emit_test_background_task.py +++ /dev/null @@ -1,12 +0,0 @@ -from django.core.management import BaseCommand - -from mail.tasks import emit_test_file - - -class Command(BaseCommand): - """ - pipenv run ./manage.py emit_test_background_task - """ - - def handle(self, *args, **options): - emit_test_file() diff --git a/mail/tasks.py b/mail/tasks.py deleted file mode 100644 index 10bd1073..00000000 --- a/mail/tasks.py +++ /dev/null @@ -1,17 +0,0 @@ -import logging -import os - -from background_task import background -from django.conf import settings - -logger = logging.getLogger(__name__) - -NOTIFY_USERS_TASK_QUEUE = "notify_users_queue" -LICENCE_DATA_TASK_QUEUE = "licences_updates_queue" - - -@background(queue="test_queue", schedule=0) -def emit_test_file(): - test_file_path = os.path.join(settings.BASE_DIR, ".background-tasks-is-ready") - with open(test_file_path, "w") as test_file: - test_file.write("OK") diff --git a/mail/tests/test_apps.py b/mail/tests/test_apps.py index 7ef2e1b6..d8eafc66 100644 --- a/mail/tests/test_apps.py +++ b/mail/tests/test_apps.py @@ -87,7 +87,7 @@ def test_app_initialization_processes_usage_data(self, put_request): ) # We expect our UsageData record to be processed as part of this initialization function - MailConfig.initialize_background_tasks() + MailConfig.initialize_send_licence_usage_figures_to_lite_api() self.usage_data.refresh_from_db() put_request.assert_called_with( diff --git a/mail/tests/test_send_licence_usage_figures_to_lite_api.py b/mail/tests/test_send_licence_usage_figures_to_lite_api.py index abc62f5e..9a89ef5e 100644 --- a/mail/tests/test_send_licence_usage_figures_to_lite_api.py +++ b/mail/tests/test_send_licence_usage_figures_to_lite_api.py @@ -30,7 +30,6 @@ def json(self): return self.json_data -@override_settings(BACKGROUND_TASK_ENABLED=False) # Disable task from being run on app initialization class UpdateUsagesTaskTests(LiteHMRCTestClient): def setUp(self): super().setUp() diff --git a/mail/tests/test_send_lite_licence_updates_task.py b/mail/tests/test_send_lite_licence_updates_task.py index 48926f8f..27f81d2b 100644 --- a/mail/tests/test_send_lite_licence_updates_task.py +++ b/mail/tests/test_send_lite_licence_updates_task.py @@ -1,8 +1,8 @@ +from smtplib import SMTPException from unittest import mock from django.test import override_settings from parameterized import parameterized -from smtplib import SMTPException from mail.celery_tasks import send_licence_details_to_hmrc from mail.enums import ReceptionStatusEnum @@ -10,7 +10,6 @@ from mail.tests.libraries.client import LiteHMRCTestClient -@override_settings(BACKGROUND_TASK_ENABLED=False) # Disable task from being run on app initialization class TaskTests(LiteHMRCTestClient): def setUp(self): super().setUp() diff --git a/mock_hmrc/apps.py b/mock_hmrc/apps.py index b2268281..ed9d5b0c 100644 --- a/mock_hmrc/apps.py +++ b/mock_hmrc/apps.py @@ -6,8 +6,8 @@ class MockHmrcConfig(AppConfig): name = "mock_hmrc" @classmethod - def initialize_background_tasks(cls, **kwargs): + def initialize_send_licence_usage_figures_to_lite_api(cls, **kwargs): pass def ready(self): - post_migrate.connect(self.initialize_background_tasks, sender=self) + post_migrate.connect(self.initialize_send_licence_usage_figures_to_lite_api, sender=self) diff --git a/mock_hmrc/tasks.py b/mock_hmrc/tasks.py index 7bba7f9d..0138af66 100644 --- a/mock_hmrc/tasks.py +++ b/mock_hmrc/tasks.py @@ -1,11 +1,8 @@ import logging -from background_task import background - from mock_hmrc.handler import parse_and_reply_emails -@background(queue="handle_hmrc_replies_queue", schedule=0) def handle_replies(): logging.info("Polling mock HMRC inbox for updates") diff --git a/mock_hmrc/views.py b/mock_hmrc/views.py index 0636fc68..dc4fc63c 100644 --- a/mock_hmrc/views.py +++ b/mock_hmrc/views.py @@ -9,6 +9,6 @@ class HandleReplies(APIView): def get(self, request): - handle_replies.now() + handle_replies() return HttpResponse(status=status.HTTP_200_OK)