From 158a0e1a0aaf9a45536d6b6698d988ea17c977a1 Mon Sep 17 00:00:00 2001 From: Gurdeep Atwal Date: Tue, 30 Apr 2024 17:50:54 +0100 Subject: [PATCH 01/73] disable ES indexing --- api/conf/settings.py | 2 +- .../migrations/0024_denials_data_migration.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/api/conf/settings.py b/api/conf/settings.py index 45988f0227..bc0cdfa5e8 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -342,7 +342,7 @@ def _build_redis_url(base_url, db_number, **query_args): } ENABLE_SPIRE_SEARCH = env.bool("ENABLE_SPIRE_SEARCH", False) - + ELASTICSEARCH_DSL_AUTOSYNC = True ELASTICSEARCH_PRODUCT_INDEXES = {"LITE": ELASTICSEARCH_PRODUCT_INDEX_ALIAS} ELASTICSEARCH_APPLICATION_INDEXES = {"LITE": ELASTICSEARCH_APPLICATION_INDEX_ALIAS} SPIRE_APPLICATION_INDEX_NAME = env.str("SPIRE_APPLICATION_INDEX_NAME", "spire-application-alias") diff --git a/api/external_data/migrations/0024_denials_data_migration.py b/api/external_data/migrations/0024_denials_data_migration.py index 4a7bfafec6..c579082dbe 100644 --- a/api/external_data/migrations/0024_denials_data_migration.py +++ b/api/external_data/migrations/0024_denials_data_migration.py @@ -6,7 +6,7 @@ from django.db.models import Q from django.db import IntegrityError from django.db import transaction - +from django.conf import settings log = logging.getLogger(__name__) required_fields = [ @@ -22,6 +22,8 @@ ] def denials_data_migration(apps, schema_editor): + settings.ELASTICSEARCH_DSL_AUTOSYNC = False + DenialEntity = apps.get_model("external_data", "DenialEntity") Denial = apps.get_model("external_data", "Denial") @@ -54,8 +56,10 @@ def denials_data_migration(apps, schema_editor): ) transaction.savepoint_rollback(sid) else: - transaction.savepoint_commit(sid) - + transaction.savepoint_commit(sid) + + settings.ELASTICSEARCH_DSL_AUTOSYNC = True + class Migration(migrations.Migration): dependencies = [ ("external_data", "0023_set_denial_entity_type"), From 090f3d0dee5ea321daf6522a80f00c4d7589b836 Mon Sep 17 00:00:00 2001 From: Kevin Carrogan Date: Wed, 1 May 2024 10:12:28 +0100 Subject: [PATCH 02/73] Use bulk updates and creates for denials migrations --- .../migrations/0023_set_denial_entity_type.py | 42 ++++----- .../migrations/0024_denials_data_migration.py | 86 ++++++++++--------- 2 files changed, 69 insertions(+), 59 deletions(-) diff --git a/api/external_data/migrations/0023_set_denial_entity_type.py b/api/external_data/migrations/0023_set_denial_entity_type.py index 253ad64833..82550e7940 100644 --- a/api/external_data/migrations/0023_set_denial_entity_type.py +++ b/api/external_data/migrations/0023_set_denial_entity_type.py @@ -5,38 +5,40 @@ def get_denial_entity_type(data): + if not isinstance(data, dict): + return - if isinstance(data, dict): - entity_type = "" - normalised_entity_type_dict = {keys.lower(): values.lower() for keys, values in data.items()} + entity_type = "" + normalised_entity_type_dict = {keys.lower(): values.lower() for keys, values in data.items()} - is_end_user_flag = normalised_entity_type_dict.get("end_user_flag", "false") == "true" - is_consignee_flag = normalised_entity_type_dict.get("consignee_flag", "false") == "true" - is_other_role = len(normalised_entity_type_dict.get("other_role", "")) > 0 + is_end_user_flag = normalised_entity_type_dict.get("end_user_flag", "false") == "true" + is_consignee_flag = normalised_entity_type_dict.get("consignee_flag", "false") == "true" + is_other_role = len(normalised_entity_type_dict.get("other_role", "")) > 0 - if is_end_user_flag and is_consignee_flag: - entity_type = DenialEntityType.END_USER - elif not is_end_user_flag and is_consignee_flag: - entity_type = DenialEntityType.CONSIGNEE - elif is_end_user_flag and not is_consignee_flag: - entity_type = DenialEntityType.END_USER - elif not is_end_user_flag and not is_consignee_flag and is_other_role: - entity_type = DenialEntityType.THIRD_PARTY + if is_end_user_flag and is_consignee_flag: + entity_type = DenialEntityType.END_USER + elif not is_end_user_flag and is_consignee_flag: + entity_type = DenialEntityType.CONSIGNEE + elif is_end_user_flag and not is_consignee_flag: + entity_type = DenialEntityType.END_USER + elif not is_end_user_flag and not is_consignee_flag and is_other_role: + entity_type = DenialEntityType.THIRD_PARTY - return entity_type + return entity_type def set_denial_entity_type(apps, schema_editor): - DenialEntity = apps.get_model("external_data", "DenialEntity") - for denial_entity in DenialEntity.objects.filter(entity_type__isnull=True): + denials_to_update = [] + for denial_entity in DenialEntity.objects.filter(entity_type__isnull=True): denial_entity_type = get_denial_entity_type(denial_entity.data) - - if denial_entity_type in ["end_user", "consignee", "third_party"]: + if denial_entity_type is not None and denial_entity_type in ["end_user", "consignee", "third_party"]: denial_entity.entity_type = denial_entity_type - denial_entity.save() + denials_to_update.append(denial_entity) + + DenialEntity.objects.bulk_update(denials_to_update, ["entity_type"]) class Migration(migrations.Migration): diff --git a/api/external_data/migrations/0024_denials_data_migration.py b/api/external_data/migrations/0024_denials_data_migration.py index c579082dbe..7a6dc081c0 100644 --- a/api/external_data/migrations/0024_denials_data_migration.py +++ b/api/external_data/migrations/0024_denials_data_migration.py @@ -1,65 +1,73 @@ # Generated by Django 4.2.10 on 2024-04-02 14:59 import logging -from django.db import migrations -from django.forms.models import model_to_dict + +from django.db import IntegrityError, migrations, transaction from django.db.models import Q -from django.db import IntegrityError -from django.db import transaction -from django.conf import settings +from django.forms.models import model_to_dict + + log = logging.getLogger(__name__) + required_fields = [ - "reference", - "regime_reg_ref", - "notifying_government", - "item_list_codes", - "item_description", - "end_use", - "is_revoked", - "is_revoked_comment", - "reason_for_refusal", + "reference", + "regime_reg_ref", + "notifying_government", + "item_list_codes", + "item_description", + "end_use", + "is_revoked", + "is_revoked_comment", + "reason_for_refusal", ] -def denials_data_migration(apps, schema_editor): - settings.ELASTICSEARCH_DSL_AUTOSYNC = False +def denials_data_migration(apps, schema_editor): DenialEntity = apps.get_model("external_data", "DenialEntity") Denial = apps.get_model("external_data", "Denial") # There are a handfull (10) of regime_reg_refs that are null which was in the initial load - # Assumption here is that they can be deleted since it's erroneous data as we now know - # regime_reg_ref is considered a unique DN record. + # Assumption here is that they can be deleted since it's erroneous data as we now know + # regime_reg_ref is considered a unique DN record. - total_null_regime_reg_ref = DenialEntity.objects.filter(Q(regime_reg_ref__isnull=True) | Q(regime_reg_ref='')) + total_null_regime_reg_ref = DenialEntity.objects.filter(Q(regime_reg_ref__isnull=True) | Q(regime_reg_ref="")) log.info( - "Delete null regime_reg_ref total -> %s", - total_null_regime_reg_ref.count(), + "Delete null regime_reg_ref total -> %s", + total_null_regime_reg_ref.count(), ) total_null_regime_reg_ref.delete() - duplicate_denial_errors = [] + with transaction.atomic(): sid = transaction.savepoint() + denials_to_create = [] + denials_to_update = [] for denial_entity in DenialEntity.objects.all(): + denial_entity_dict = { + key: value for (key, value) in model_to_dict(denial_entity).items() if key in required_fields + } + try: - denial_entity_dict = {key:value for (key,value) in model_to_dict(denial_entity).items() if key in required_fields} - denial , _ = Denial.objects.get_or_create(**denial_entity_dict) - denial_entity.denial = denial - denial_entity.save() - except IntegrityError as e: - duplicate_denial_errors.append(denial_entity.regime_reg_ref) - - if duplicate_denial_errors: - log.info( - "There are the following duplicate denials in the database rolling back this migration: -> %s", - duplicate_denial_errors, - ) + denial = Denial.objects.get(**denial_entity_dict) + except Denial.DoesNotExist: + denial = Denial(**denial_entity_dict) + denials_to_create.append(denial) + + denial_entity.denial = denial + denials_to_update.append(denial_entity) + + try: + Denial.objects.bulk_create(denials_to_create) + except IntegrityError: + log.exception("There were errors creating denial objects") transaction.savepoint_rollback(sid) - else: - transaction.savepoint_commit(sid) - - settings.ELASTICSEARCH_DSL_AUTOSYNC = True - + return + + DenialEntity.objects.bulk_update(denials_to_update, ["denial"]) + + transaction.savepoint_commit(sid) + + class Migration(migrations.Migration): dependencies = [ ("external_data", "0023_set_denial_entity_type"), From 0099de80a63e2799898230e04f3792983aa5d390 Mon Sep 17 00:00:00 2001 From: Kevin Carrogan Date: Wed, 1 May 2024 10:56:50 +0100 Subject: [PATCH 03/73] fixup --- api/conf/settings.py | 2 +- .../migrations/0023_set_denial_entity_type.py | 42 ++++---- .../migrations/0024_denials_data_migration.py | 49 ++++------ .../tests/test_0024_denials_data_migration.py | 95 ++++++++++++------- 4 files changed, 102 insertions(+), 86 deletions(-) diff --git a/api/conf/settings.py b/api/conf/settings.py index bc0cdfa5e8..45988f0227 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -342,7 +342,7 @@ def _build_redis_url(base_url, db_number, **query_args): } ENABLE_SPIRE_SEARCH = env.bool("ENABLE_SPIRE_SEARCH", False) - ELASTICSEARCH_DSL_AUTOSYNC = True + ELASTICSEARCH_PRODUCT_INDEXES = {"LITE": ELASTICSEARCH_PRODUCT_INDEX_ALIAS} ELASTICSEARCH_APPLICATION_INDEXES = {"LITE": ELASTICSEARCH_APPLICATION_INDEX_ALIAS} SPIRE_APPLICATION_INDEX_NAME = env.str("SPIRE_APPLICATION_INDEX_NAME", "spire-application-alias") diff --git a/api/external_data/migrations/0023_set_denial_entity_type.py b/api/external_data/migrations/0023_set_denial_entity_type.py index 82550e7940..253ad64833 100644 --- a/api/external_data/migrations/0023_set_denial_entity_type.py +++ b/api/external_data/migrations/0023_set_denial_entity_type.py @@ -5,40 +5,38 @@ def get_denial_entity_type(data): - if not isinstance(data, dict): - return - entity_type = "" - normalised_entity_type_dict = {keys.lower(): values.lower() for keys, values in data.items()} + if isinstance(data, dict): + entity_type = "" + normalised_entity_type_dict = {keys.lower(): values.lower() for keys, values in data.items()} - is_end_user_flag = normalised_entity_type_dict.get("end_user_flag", "false") == "true" - is_consignee_flag = normalised_entity_type_dict.get("consignee_flag", "false") == "true" - is_other_role = len(normalised_entity_type_dict.get("other_role", "")) > 0 + is_end_user_flag = normalised_entity_type_dict.get("end_user_flag", "false") == "true" + is_consignee_flag = normalised_entity_type_dict.get("consignee_flag", "false") == "true" + is_other_role = len(normalised_entity_type_dict.get("other_role", "")) > 0 - if is_end_user_flag and is_consignee_flag: - entity_type = DenialEntityType.END_USER - elif not is_end_user_flag and is_consignee_flag: - entity_type = DenialEntityType.CONSIGNEE - elif is_end_user_flag and not is_consignee_flag: - entity_type = DenialEntityType.END_USER - elif not is_end_user_flag and not is_consignee_flag and is_other_role: - entity_type = DenialEntityType.THIRD_PARTY + if is_end_user_flag and is_consignee_flag: + entity_type = DenialEntityType.END_USER + elif not is_end_user_flag and is_consignee_flag: + entity_type = DenialEntityType.CONSIGNEE + elif is_end_user_flag and not is_consignee_flag: + entity_type = DenialEntityType.END_USER + elif not is_end_user_flag and not is_consignee_flag and is_other_role: + entity_type = DenialEntityType.THIRD_PARTY - return entity_type + return entity_type def set_denial_entity_type(apps, schema_editor): - DenialEntity = apps.get_model("external_data", "DenialEntity") - denials_to_update = [] + DenialEntity = apps.get_model("external_data", "DenialEntity") for denial_entity in DenialEntity.objects.filter(entity_type__isnull=True): + denial_entity_type = get_denial_entity_type(denial_entity.data) - if denial_entity_type is not None and denial_entity_type in ["end_user", "consignee", "third_party"]: - denial_entity.entity_type = denial_entity_type - denials_to_update.append(denial_entity) - DenialEntity.objects.bulk_update(denials_to_update, ["entity_type"]) + if denial_entity_type in ["end_user", "consignee", "third_party"]: + denial_entity.entity_type = denial_entity_type + denial_entity.save() class Migration(migrations.Migration): diff --git a/api/external_data/migrations/0024_denials_data_migration.py b/api/external_data/migrations/0024_denials_data_migration.py index 7a6dc081c0..5f18ea5f22 100644 --- a/api/external_data/migrations/0024_denials_data_migration.py +++ b/api/external_data/migrations/0024_denials_data_migration.py @@ -38,35 +38,26 @@ def denials_data_migration(apps, schema_editor): ) total_null_regime_reg_ref.delete() - with transaction.atomic(): - sid = transaction.savepoint() - denials_to_create = [] - denials_to_update = [] - for denial_entity in DenialEntity.objects.all(): - denial_entity_dict = { - key: value for (key, value) in model_to_dict(denial_entity).items() if key in required_fields - } - - try: - denial = Denial.objects.get(**denial_entity_dict) - except Denial.DoesNotExist: - denial = Denial(**denial_entity_dict) - denials_to_create.append(denial) - - denial_entity.denial = denial - denials_to_update.append(denial_entity) - - try: - Denial.objects.bulk_create(denials_to_create) - except IntegrityError: - log.exception("There were errors creating denial objects") - transaction.savepoint_rollback(sid) - return - - DenialEntity.objects.bulk_update(denials_to_update, ["denial"]) - - transaction.savepoint_commit(sid) - + denials_to_create = [] + denials_entitiy_to_update = [] + denials_dict_map = {} + for denial_entity in DenialEntity.objects.all(): + denial_entity_dict = { + key: value for (key, value) in model_to_dict(denial_entity).items() if key in required_fields + } + denials_dict_map[denial_entity_dict["regime_reg_ref"]] = denial_entity_dict + + for denials_dict_map_item in denials_dict_map.values(): + denial = Denial(**denials_dict_map_item) + denials_to_create.append(denial) + + Denial.objects.bulk_create(denials_to_create) + + for denial_entity in DenialEntity.objects.all(): + denial_entity.denial = Denial.objects.get(regime_reg_ref=denial_entity.regime_reg_ref) + denials_entitiy_to_update.append(denial_entity) + + DenialEntity.objects.bulk_update(denials_entitiy_to_update, ["denial"]) class Migration(migrations.Migration): dependencies = [ diff --git a/api/external_data/migrations/tests/test_0024_denials_data_migration.py b/api/external_data/migrations/tests/test_0024_denials_data_migration.py index a17be6567c..e1bfe8e0ba 100644 --- a/api/external_data/migrations/tests/test_0024_denials_data_migration.py +++ b/api/external_data/migrations/tests/test_0024_denials_data_migration.py @@ -4,11 +4,65 @@ test_data = [ -{"reference":"DN2010\/0057","regime_reg_ref":"reg.123.123","name":"name 1","address":"address 1","notifying_government":"UK","country":"UK","item_list_codes":"all","item_description":"desc a","end_use":"use 1","reason_for_refusal":"a"}, -{"reference":"DN2010\/0057","regime_reg_ref":"reg.123.1234","name":"name 2","address":"address 2","notifying_government":"UK","country":"UK","item_list_codes":"all","item_description":"desc a","end_use":"use 1","reason_for_refusal":"a"}, -{"reference":"DN2010\/0057","regime_reg_ref":"reg.123.1234","name":"name 3","address":"address 3","notifying_government":"UK","country":"UK","item_list_codes":"all","item_description":"desc a","end_use":"use 1","reason_for_refusal":"a"}, -{"reference":"DN2010\/0057","regime_reg_ref":"reg.123.1234","name":"name 4","address":"address 4","notifying_government":"UK","country":"UK","item_list_codes":"all","item_description":"desc a","end_use":"use 1","reason_for_refusal":"a"}, -{"reference":"DN2010\/0057","name":"bad record","address":"bad record","notifying_government":"UK","country":"bad","item_list_codes":"all","item_description":"bad","end_use":"bad","reason_for_refusal":"bad "} + { + "reference": "DN2010\/0057", + "regime_reg_ref": "reg.123.123", + "name": "name 1", + "address": "address 1", + "notifying_government": "UK", + "country": "UK", + "item_list_codes": "all", + "item_description": "desc a", + "end_use": "use 1", + "reason_for_refusal": "a", + }, + { + "reference": "DN2010\/0057", + "regime_reg_ref": "reg.123.1234", + "name": "name 2", + "address": "address 2", + "notifying_government": "UK", + "country": "UK", + "item_list_codes": "all", + "item_description": "desc a", + "end_use": "use 1", + "reason_for_refusal": "a", + }, + { + "reference": "DN2010\/0057", + "regime_reg_ref": "reg.123.1234", + "name": "name 3", + "address": "address 3", + "notifying_government": "UK", + "country": "UK", + "item_list_codes": "all", + "item_description": "desc a", + "end_use": "use 1", + "reason_for_refusal": "a", + }, + { + "reference": "DN2010\/0057", + "regime_reg_ref": "reg.123.1234", + "name": "name 4", + "address": "address 4", + "notifying_government": "UK", + "country": "UK", + "item_list_codes": "all", + "item_description": "desc a", + "end_use": "use 1", + "reason_for_refusal": "a", + }, + { + "reference": "DN2010\/0057", + "name": "bad record", + "address": "bad record", + "notifying_government": "UK", + "country": "bad", + "item_list_codes": "all", + "item_description": "bad", + "end_use": "bad", + "reason_for_refusal": "bad ", + }, ] @@ -18,14 +72,10 @@ class TestDenialDataMigration(MigratorTestCase): migrate_from = ("external_data", "0023_set_denial_entity_type") migrate_to = ("external_data", "0024_denials_data_migration") - def prepare(self): DenialEntity = self.old_state.apps.get_model("external_data", "DenialEntity") for row in test_data: - DenialEntity.objects.create(**row) - - - + DenialEntity.objects.create(**row) def test_0023_denials_data_migration(self): DenialEntity = self.new_state.apps.get_model("external_data", "DenialEntity") @@ -33,27 +83,4 @@ def test_0023_denials_data_migration(self): self.assertEqual(DenialEntity.objects.all().count(), 4) self.assertEqual(Denial.objects.all().count(), 2) - self.assertEqual(Denial.objects.get(regime_reg_ref='reg.123.1234').denial_entity.count(), 3) - - -@pytest.mark.django_db() -class TestDenialDataDuplicatesMigration(MigratorTestCase): - - migrate_from = ("external_data", "0023_set_denial_entity_type") - migrate_to = ("external_data", "0024_denials_data_migration") - - - def prepare(self): - DenialEntity = self.old_state.apps.get_model("external_data", "DenialEntity") - for row in test_data: - DenialEntity.objects.create(**row) - test_data[0]["end_use"] = "end_use b" - DenialEntity.objects.create(**test_data[0]) - - - - def test_0024_denials_data_migration_duplicates(self): - DenialEntity = self.new_state.apps.get_model("external_data", "DenialEntity") - Denial = self.new_state.apps.get_model("external_data", "Denial") - self.assertEqual(DenialEntity.objects.all().count(), 5) - self.assertEqual(Denial.objects.all().count(), 0) + self.assertEqual(Denial.objects.get(regime_reg_ref="reg.123.1234").denial_entity.count(), 3) From f3e25da4be55989ce2b1ea674e83109e0ae97a5f Mon Sep 17 00:00:00 2001 From: "mark.j0hnst0n" Date: Wed, 19 Jun 2024 16:12:56 +0100 Subject: [PATCH 04/73] add in asim format logging functionality and env variable to enable switching on --- Pipfile | 1 + Pipfile.lock | 131 ++++++++++++++++++++++++++----------------- api/conf/settings.py | 7 ++- 3 files changed, 88 insertions(+), 51 deletions(-) diff --git a/Pipfile b/Pipfile index 95cb68e7df..cf59bbf14b 100644 --- a/Pipfile +++ b/Pipfile @@ -73,6 +73,7 @@ django-queryable-properties = "~=1.9.1" database-sanitizer = ">=1.1.0" django-reversion = ">=5.0.12" psycopg = "~=3.1.18" +django-log-formatter-asim = "~=0.0.5" [requires] python_version = "3.9" diff --git a/Pipfile.lock b/Pipfile.lock index 27248a6040..47c4d5dfdb 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "4a26242ac9e371e24fea7168cdacc4606a5f04dea1f42ae73f41f0ba6d505e39" + "sha256": "5e8956feac774b3af51326cc1b52d81eef45978cc2d37e50f611cfa0c72f116b" }, "pipfile-spec": 6, "requires": { @@ -25,14 +25,6 @@ "markers": "python_version >= '3.6'", "version": "==5.2.0" }, - "appnope": { - "hashes": [ - "sha256:1de3860566df9caf38f01f86f65e0e13e379af54f9e4bee1e66b48f2efffd1ee", - "sha256:502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c" - ], - "markers": "sys_platform == 'darwin'", - "version": "==0.1.4" - }, "asgiref": { "hashes": [ "sha256:3e1e3ecc849832fe52ccf2cb6686b7a55f82bb1d6aee72a58826471390335e47", @@ -64,31 +56,6 @@ ], "version": "==0.2.0" }, - "backports.zoneinfo": { - "extras": [ - "tzdata" - ], - "hashes": [ - "sha256:17746bd546106fa389c51dbea67c8b7c8f0d14b5526a579ca6ccf5ed72c526cf", - "sha256:1b13e654a55cd45672cb54ed12148cd33628f672548f373963b0bff67b217328", - "sha256:1c5742112073a563c81f786e77514969acb58649bcdf6cdf0b4ed31a348d4546", - "sha256:4a0f800587060bf8880f954dbef70de6c11bbe59c673c3d818921f042f9954a6", - "sha256:5c144945a7752ca544b4b78c8c41544cdfaf9786f25fe5ffb10e838e19a27570", - "sha256:7b0a64cda4145548fed9efc10322770f929b944ce5cee6c0dfe0c87bf4c0c8c9", - "sha256:8439c030a11780786a2002261569bdf362264f605dfa4d65090b64b05c9f79a7", - "sha256:8961c0f32cd0336fb8e8ead11a1f8cd99ec07145ec2931122faaac1c8f7fd987", - "sha256:89a48c0d158a3cc3f654da4c2de1ceba85263fafb861b98b59040a5086259722", - "sha256:a76b38c52400b762e48131494ba26be363491ac4f9a04c1b7e92483d169f6582", - "sha256:da6013fd84a690242c310d77ddb8441a559e9cb3d3d59ebac9aca1a57b2e18bc", - "sha256:e55b384612d93be96506932a786bbcde5a2db7a9e6a4bb4bffe8b733f5b9036b", - "sha256:e81b76cace8eda1fca50e345242ba977f9be6ae3945af8d46326d776b4cf78d1", - "sha256:e8236383a20872c0cdf5a62b554b27538db7fa1bbec52429d8d106effbaeca08", - "sha256:f04e857b59d9d1ccc39ce2da1021d196e47234873820cbeaad210724b1ee28ac", - "sha256:fadbfe37f74051d024037f223b8e001611eac868b5c5b06144ef4d8b799862f2" - ], - "markers": "python_version < '3.9'", - "version": "==0.2.1" - }, "billiard": { "hashes": [ "sha256:07aa978b308f334ff8282bd4a746e681b3513db5c9a514cbdd810cbbdc19714d", @@ -205,6 +172,14 @@ "markers": "python_version >= '3.8'", "version": "==1.16.0" }, + "cfgv": { + "hashes": [ + "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9", + "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560" + ], + "markers": "python_version >= '3.8'", + "version": "==3.4.0" + }, "charset-normalizer": { "hashes": [ "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027", @@ -412,6 +387,13 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==0.6.0" }, + "distlib": { + "hashes": [ + "sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784", + "sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64" + ], + "version": "==0.3.8" + }, "django": { "hashes": [ "sha256:837e3cf1f6c31347a1396a3f6b65688f2b4bb4a11c580dcb628b5afe527b68a5", @@ -498,6 +480,15 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", "version": "==3.0.7" }, + "django-log-formatter-asim": { + "hashes": [ + "sha256:77501044786d8f5f249ed90f8eca0b7c79e57003519373dc2c69006e546daf3b", + "sha256:cfa38573db9c973d672b3d517a27994e8b1d794bbdebb8b5ddabe63be02c8af5" + ], + "index": "pypi", + "markers": "python_version >= '3.9' and python_version < '4'", + "version": "==0.0.5" + }, "django-log-formatter-ecs": { "hashes": [ "sha256:1e8731dd25a11ac64e789f19931e12fe7ef8ad1a172b7bceb2ea5cab185a583e", @@ -677,6 +668,14 @@ "markers": "python_version >= '3.8'", "version": "==23.2.1" }, + "filelock": { + "hashes": [ + "sha256:58a2549afdf9e02e10720eaa4d4470f56386d7a6f72edd7d0596337af8ed7ad8", + "sha256:71b3102950e91dfc1bb4209b64be4dc8854f40e5f534428d8684f953ac847fac" + ], + "markers": "python_version >= '3.8'", + "version": "==3.15.1" + }, "future": { "hashes": [ "sha256:929292d34f5872e70396626ef385ec22355a1fae8ad29e1a734c3e43f9fbc216", @@ -821,6 +820,14 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==1.1" }, + "identify": { + "hashes": [ + "sha256:37d93f380f4de590500d9dba7db359d0d3da95ffe7f9de1753faa159e71e7dfa", + "sha256:e5e00f54165f9047fbebeb4a560f9acfb8af4c88232be60a488e9b68d122745d" + ], + "markers": "python_version >= '3.8'", + "version": "==2.5.36" + }, "idna": { "hashes": [ "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc", @@ -925,6 +932,14 @@ ], "version": "==1.6" }, + "nodeenv": { + "hashes": [ + "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f", + "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'", + "version": "==1.9.1" + }, "notifications-python-client": { "hashes": [ "sha256:3193a74eba1a5d33aac1662e57068de9705eb63396eed4708d81a6bdbb07b5ee" @@ -1063,6 +1078,22 @@ "markers": "python_version >= '3.8'", "version": "==10.2.0" }, + "platformdirs": { + "hashes": [ + "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee", + "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3" + ], + "markers": "python_version >= '3.8'", + "version": "==4.2.2" + }, + "pre-commit": { + "hashes": [ + "sha256:8ca3ad567bc78a4972a3f1a477e94a79d4597e8140a6e0b651c5e33899c3654a", + "sha256:fae36fd1d7ad7d6a5a1c0b0d5adb2ed1a3bda5a21bf6c3e5372073d7a11cd4c5" + ], + "markers": "python_version >= '3.9'", + "version": "==3.7.1" + }, "prompt-toolkit": { "hashes": [ "sha256:0d7bfa67001d5e39d02c224b663abc33687405033a8c422d0d675a5a13361d10", @@ -1260,11 +1291,11 @@ }, "setuptools": { "hashes": [ - "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4", - "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0" + "sha256:01a1e793faa5bd89abc851fa15d0a0db26f160890c7102cd8dce643e886b47f5", + "sha256:d9b8b771455a97c8a9f3ab3448ebe0b29b5e105f1228bba41028be116985a267" ], "markers": "python_version >= '3.8'", - "version": "==70.0.0" + "version": "==70.1.0" }, "six": { "hashes": [ @@ -1344,7 +1375,7 @@ "sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3", "sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "markers": "python_version >= '3.6'", "version": "==1.26.19" }, "vine": { @@ -1355,6 +1386,14 @@ "markers": "python_version >= '3.6'", "version": "==5.1.0" }, + "virtualenv": { + "hashes": [ + "sha256:82bf0f4eebbb78d36ddaee0283d43fe5736b53880b8a8cdcd37390a07ac3741c", + "sha256:a624db5e94f01ad993d476b9ee5346fdf7b9de43ccaee0e0197012dc838a0e9b" + ], + "markers": "python_version >= '3.7'", + "version": "==20.26.2" + }, "wcwidth": { "hashes": [ "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859", @@ -1464,14 +1503,6 @@ } }, "develop": { - "appnope": { - "hashes": [ - "sha256:1de3860566df9caf38f01f86f65e0e13e379af54f9e4bee1e66b48f2efffd1ee", - "sha256:502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c" - ], - "markers": "sys_platform == 'darwin'", - "version": "==0.1.4" - }, "astroid": { "hashes": [ "sha256:71ea07f44df9568a75d0f354c49143a4575d90645e9fead6dfb52c26a85ed13a", @@ -2440,11 +2471,11 @@ }, "setuptools": { "hashes": [ - "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4", - "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0" + "sha256:01a1e793faa5bd89abc851fa15d0a0db26f160890c7102cd8dce643e886b47f5", + "sha256:d9b8b771455a97c8a9f3ab3448ebe0b29b5e105f1228bba41028be116985a267" ], "markers": "python_version >= '3.8'", - "version": "==70.0.0" + "version": "==70.1.0" }, "six": { "hashes": [ @@ -2514,7 +2545,7 @@ "sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3", "sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "markers": "python_version >= '3.6'", "version": "==1.26.19" }, "watchdog": { diff --git a/api/conf/settings.py b/api/conf/settings.py index bd78dbb0a6..8a8186bd4d 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -8,6 +8,7 @@ import sentry_sdk from sentry_sdk.integrations.django import DjangoIntegration from django_log_formatter_ecs import ECSFormatter +from django_log_formatter_asim import ASIMFormatter from django.urls import reverse_lazy @@ -369,17 +370,21 @@ def _build_redis_url(base_url, db_number, **query_args): "formatters": { "simple": {"format": "{asctime} {levelname} {message}", "style": "{"}, "ecs_formatter": {"()": ECSFormatter}, + "asim_formatter": {"()": ASIMFormatter}, }, "handlers": { "stdout": {"class": "logging.StreamHandler", "formatter": "simple"}, "ecs": {"class": "logging.StreamHandler", "formatter": "ecs_formatter"}, "sentry": {"class": "sentry_sdk.integrations.logging.EventHandler"}, + "asim": {"class": "logging.StreamHandler", "formatter": "asim_formatter"}, }, - "root": {"handlers": ["stdout", "ecs"], "level": env("LOG_LEVEL").upper()}, + "root": {"handlers": ["stdout", "ecs", "asim"], "level": env("LOG_LEVEL").upper()}, "loggers": { DENIAL_REASONS_DELETION_LOGGER: {"handlers": ["sentry"], "level": logging.WARNING}, }, } +if additional_logger_config := env.json("ADDITIONAL_LOGGER_CONFIG", default=None): + LOGGING["loggers"] = additional_logger_config # Sentry if env.str("SENTRY_DSN", ""): From 1aa92af049a81f2d1c8a7438dd3fd33dedcf79a4 Mon Sep 17 00:00:00 2001 From: "mark.j0hnst0n" Date: Wed, 19 Jun 2024 17:02:41 +0100 Subject: [PATCH 05/73] removal of walrus operator --- api/conf/settings.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/conf/settings.py b/api/conf/settings.py index 8a8186bd4d..6ad9d9dd03 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -383,7 +383,9 @@ def _build_redis_url(base_url, db_number, **query_args): DENIAL_REASONS_DELETION_LOGGER: {"handlers": ["sentry"], "level": logging.WARNING}, }, } -if additional_logger_config := env.json("ADDITIONAL_LOGGER_CONFIG", default=None): + +additional_logger_config = env.json("ADDITIONAL_LOGGER_CONFIG", default=None) +if additional_logger_config: LOGGING["loggers"] = additional_logger_config # Sentry From 90fd46a48fa4c59760a403393d8d9f0fd243cfde Mon Sep 17 00:00:00 2001 From: "mark.j0hnst0n" Date: Thu, 20 Jun 2024 11:42:08 +0100 Subject: [PATCH 06/73] remove env variable --- api/conf/settings.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/api/conf/settings.py b/api/conf/settings.py index 6ad9d9dd03..c99c9afeaf 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -384,9 +384,6 @@ def _build_redis_url(base_url, db_number, **query_args): }, } -additional_logger_config = env.json("ADDITIONAL_LOGGER_CONFIG", default=None) -if additional_logger_config: - LOGGING["loggers"] = additional_logger_config # Sentry if env.str("SENTRY_DSN", ""): From 70bf67a708f05f1df5388704df553b6cf374ac28 Mon Sep 17 00:00:00 2001 From: "mark.j0hnst0n" Date: Thu, 20 Jun 2024 15:26:18 +0100 Subject: [PATCH 07/73] split logs for local, gov paas and dbt platform --- api/conf/settings.py | 45 +++++++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/api/conf/settings.py b/api/conf/settings.py index c99c9afeaf..3fd2fa2487 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -363,27 +363,46 @@ def _build_redis_url(base_url, db_number, **query_args): DENIAL_REASONS_DELETION_LOGGER = "denial_reasons_deletion_logger" - LOGGING = { "version": 1, "disable_existing_loggers": False, - "formatters": { - "simple": {"format": "{asctime} {levelname} {message}", "style": "{"}, - "ecs_formatter": {"()": ECSFormatter}, - "asim_formatter": {"()": ASIMFormatter}, - }, - "handlers": { - "stdout": {"class": "logging.StreamHandler", "formatter": "simple"}, - "ecs": {"class": "logging.StreamHandler", "formatter": "ecs_formatter"}, - "sentry": {"class": "sentry_sdk.integrations.logging.EventHandler"}, - "asim": {"class": "logging.StreamHandler", "formatter": "asim_formatter"}, - }, - "root": {"handlers": ["stdout", "ecs", "asim"], "level": env("LOG_LEVEL").upper()}, "loggers": { DENIAL_REASONS_DELETION_LOGGER: {"handlers": ["sentry"], "level": logging.WARNING}, }, } +ENVIRONMENT = env.str("ENV", "") + +LOGGING = {"version": 1, "disable_existing_loggers": False} + +if ENVIRONMENT == "local": + LOGGING["formatters"] = { + "simple": {"format": "{asctime} {levelname} {message}", "style": "{"}, + } + LOGGING["handlers"] = { + "stdout": {"class": "logging.StreamHandler", "formatter": "simple"}, + } + LOGGING["root"] = {"handlers": ["stdout"], "level": env.str("LOG_LEVEL", "info").upper()} + +elif VCAP_SERVICES: + LOGGING["formatters"] = { + "ecs_formatter": {"()": ECSFormatter}, + } + LOGGING["handlers"] = { + "ecs": {"class": "logging.StreamHandler", "formatter": "ecs_formatter"}, + } + LOGGING["root"] = {"handlers": ["ecs"], "level": env.str("LOG_LEVEL", "info").upper()} + +else: + LOGGING["formatters"] = { + "asim_formatter": { + "()": ASIMFormatter, + }, + } + LOGGING["handlers"] = { + "asim": {"class": "logging.StreamHandler", "formatter": "asim_formatter"}, + } + LOGGING["root"] = {"handlers": ["asim"], "level": env.str("LOG_LEVEL", "info").upper()} # Sentry if env.str("SENTRY_DSN", ""): From 9fce6ee372483403ef7505bf0272a18614cbc494 Mon Sep 17 00:00:00 2001 From: "mark.j0hnst0n" Date: Thu, 20 Jun 2024 16:45:52 +0100 Subject: [PATCH 08/73] early return favoured and root handler formatted --- api/conf/settings.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/api/conf/settings.py b/api/conf/settings.py index 3fd2fa2487..c4e6bbcf05 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -363,9 +363,14 @@ def _build_redis_url(base_url, db_number, **query_args): DENIAL_REASONS_DELETION_LOGGER = "denial_reasons_deletion_logger" +LOGGING_ROOT_HANDLER = "asim" + LOGGING = { "version": 1, "disable_existing_loggers": False, + "formatters": {"asim_formatter": {"()": ASIMFormatter}}, + "handlers": {"asim": {"class": "logging.StreamHandler", "formatter": "asim_formatter"}}, + "root": {"handlers": [LOGGING_ROOT_HANDLER], "level": env("LOG_LEVEL").upper()}, "loggers": { DENIAL_REASONS_DELETION_LOGGER: {"handlers": ["sentry"], "level": logging.WARNING}, }, @@ -382,7 +387,7 @@ def _build_redis_url(base_url, db_number, **query_args): LOGGING["handlers"] = { "stdout": {"class": "logging.StreamHandler", "formatter": "simple"}, } - LOGGING["root"] = {"handlers": ["stdout"], "level": env.str("LOG_LEVEL", "info").upper()} + LOGGING_ROOT_HANDLER = "stdout" elif VCAP_SERVICES: LOGGING["formatters"] = { @@ -391,18 +396,8 @@ def _build_redis_url(base_url, db_number, **query_args): LOGGING["handlers"] = { "ecs": {"class": "logging.StreamHandler", "formatter": "ecs_formatter"}, } - LOGGING["root"] = {"handlers": ["ecs"], "level": env.str("LOG_LEVEL", "info").upper()} + LOGGING_ROOT_HANDLER = "ecs" -else: - LOGGING["formatters"] = { - "asim_formatter": { - "()": ASIMFormatter, - }, - } - LOGGING["handlers"] = { - "asim": {"class": "logging.StreamHandler", "formatter": "asim_formatter"}, - } - LOGGING["root"] = {"handlers": ["asim"], "level": env.str("LOG_LEVEL", "info").upper()} # Sentry if env.str("SENTRY_DSN", ""): From ab5508a36397b96d3bd3a94a4dbf7439edc01632 Mon Sep 17 00:00:00 2001 From: "mark.j0hnst0n" Date: Thu, 20 Jun 2024 16:54:33 +0100 Subject: [PATCH 09/73] include sentry handling again --- api/conf/settings.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/api/conf/settings.py b/api/conf/settings.py index c4e6bbcf05..6f329ab0eb 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -369,7 +369,10 @@ def _build_redis_url(base_url, db_number, **query_args): "version": 1, "disable_existing_loggers": False, "formatters": {"asim_formatter": {"()": ASIMFormatter}}, - "handlers": {"asim": {"class": "logging.StreamHandler", "formatter": "asim_formatter"}}, + "handlers": { + "asim": {"class": "logging.StreamHandler", "formatter": "asim_formatter"}, + "sentry": {"class": "sentry_sdk.integrations.logging.EventHandler"}, + }, "root": {"handlers": [LOGGING_ROOT_HANDLER], "level": env("LOG_LEVEL").upper()}, "loggers": { DENIAL_REASONS_DELETION_LOGGER: {"handlers": ["sentry"], "level": logging.WARNING}, @@ -395,6 +398,7 @@ def _build_redis_url(base_url, db_number, **query_args): } LOGGING["handlers"] = { "ecs": {"class": "logging.StreamHandler", "formatter": "ecs_formatter"}, + "sentry": {"class": "sentry_sdk.integrations.logging.EventHandler"}, } LOGGING_ROOT_HANDLER = "ecs" From 99d0fb0d2f7acf44ee5878538344749cb956c21b Mon Sep 17 00:00:00 2001 From: "mark.j0hnst0n" Date: Thu, 20 Jun 2024 18:46:27 +0100 Subject: [PATCH 10/73] working locally --- api/conf/settings.py | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/api/conf/settings.py b/api/conf/settings.py index 6f329ab0eb..08ade14662 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -362,46 +362,47 @@ def _build_redis_url(base_url, db_number, **query_args): DENIAL_REASONS_DELETION_LOGGER = "denial_reasons_deletion_logger" - -LOGGING_ROOT_HANDLER = "asim" +LOGGING_ROOT_HANDLER = "stdout" LOGGING = { "version": 1, "disable_existing_loggers": False, - "formatters": {"asim_formatter": {"()": ASIMFormatter}}, - "handlers": { - "asim": {"class": "logging.StreamHandler", "formatter": "asim_formatter"}, - "sentry": {"class": "sentry_sdk.integrations.logging.EventHandler"}, - }, + "handlers": {"sentry": {"level": logging.ERROR, "class": "sentry_sdk.integrations.logging.EventHandler"}}, "root": {"handlers": [LOGGING_ROOT_HANDLER], "level": env("LOG_LEVEL").upper()}, "loggers": { - DENIAL_REASONS_DELETION_LOGGER: {"handlers": ["sentry"], "level": logging.WARNING}, + DENIAL_REASONS_DELETION_LOGGER: { + "handlers": ["sentry"], + "level": logging.WARNING, + }, }, } ENVIRONMENT = env.str("ENV", "") -LOGGING = {"version": 1, "disable_existing_loggers": False} - if ENVIRONMENT == "local": + LOGGING_ROOT_HANDLER = "stdout" LOGGING["formatters"] = { "simple": {"format": "{asctime} {levelname} {message}", "style": "{"}, } - LOGGING["handlers"] = { - "stdout": {"class": "logging.StreamHandler", "formatter": "simple"}, - } - LOGGING_ROOT_HANDLER = "stdout" + LOGGING["handlers"]["stdout"] = {"class": "logging.StreamHandler", "formatter": "simple"} + elif VCAP_SERVICES: + LOGGING_ROOT_HANDLER = "ecs" LOGGING["formatters"] = { "ecs_formatter": {"()": ECSFormatter}, } - LOGGING["handlers"] = { - "ecs": {"class": "logging.StreamHandler", "formatter": "ecs_formatter"}, - "sentry": {"class": "sentry_sdk.integrations.logging.EventHandler"}, - } - LOGGING_ROOT_HANDLER = "ecs" + LOGGING["handlers"]["ecs"] = {"class": "logging.StreamHandler", "formatter": "ecs_formatter"} + +else: + LOGGING_ROOT_HANDLER = "asim" + LOGGING["formatters"] = { + "asim_formatter": { + "()": ASIMFormatter, + }, + } + LOGGING["handlers"]["asim"] = {"class": "logging.StreamHandler", "formatter": "asim_formatter"} # Sentry if env.str("SENTRY_DSN", ""): From 4ac1574d580c23a0c8debc5b8e6838e76590a236 Mon Sep 17 00:00:00 2001 From: "mark.j0hnst0n" Date: Thu, 20 Jun 2024 19:12:08 +0100 Subject: [PATCH 11/73] working locally with sentry added --- api/conf/settings.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/api/conf/settings.py b/api/conf/settings.py index 08ade14662..87ccd83e19 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -362,18 +362,15 @@ def _build_redis_url(base_url, db_number, **query_args): DENIAL_REASONS_DELETION_LOGGER = "denial_reasons_deletion_logger" -LOGGING_ROOT_HANDLER = "stdout" LOGGING = { "version": 1, "disable_existing_loggers": False, - "handlers": {"sentry": {"level": logging.ERROR, "class": "sentry_sdk.integrations.logging.EventHandler"}}, - "root": {"handlers": [LOGGING_ROOT_HANDLER], "level": env("LOG_LEVEL").upper()}, + "handlers": { + "sentry": {"class": "sentry_sdk.integrations.logging.EventHandler"}, + }, "loggers": { - DENIAL_REASONS_DELETION_LOGGER: { - "handlers": ["sentry"], - "level": logging.WARNING, - }, + DENIAL_REASONS_DELETION_LOGGER: {"handlers": ["sentry"], "level": logging.WARNING}, }, } @@ -384,16 +381,22 @@ def _build_redis_url(base_url, db_number, **query_args): LOGGING["formatters"] = { "simple": {"format": "{asctime} {levelname} {message}", "style": "{"}, } - LOGGING["handlers"]["stdout"] = {"class": "logging.StreamHandler", "formatter": "simple"} - + LOGGING["handlers"] = { + "stdout": {"class": "logging.StreamHandler", "formatter": "simple"}, + "sentry": {"class": "sentry_sdk.integrations.logging.EventHandler"}, + } + LOGGING["root"] = {"handlers": [LOGGING_ROOT_HANDLER], "level": env("LOG_LEVEL").upper()} elif VCAP_SERVICES: LOGGING_ROOT_HANDLER = "ecs" LOGGING["formatters"] = { "ecs_formatter": {"()": ECSFormatter}, } - LOGGING["handlers"]["ecs"] = {"class": "logging.StreamHandler", "formatter": "ecs_formatter"} - + LOGGING["handlers"] = { + "ecs": {"class": "logging.StreamHandler", "formatter": "ecs_formatter"}, + "sentry": {"class": "sentry_sdk.integrations.logging.EventHandler"}, + } + LOGGING["root"] = {"handlers": [LOGGING_ROOT_HANDLER], "level": env("LOG_LEVEL").upper()} else: LOGGING_ROOT_HANDLER = "asim" @@ -402,7 +405,11 @@ def _build_redis_url(base_url, db_number, **query_args): "()": ASIMFormatter, }, } - LOGGING["handlers"]["asim"] = {"class": "logging.StreamHandler", "formatter": "asim_formatter"} + LOGGING["handlers"] = { + "asim": {"class": "logging.StreamHandler", "formatter": "asim_formatter"}, + "sentry": {"class": "sentry_sdk.integrations.logging.EventHandler"}, + } + LOGGING["root"] = {"handlers": [LOGGING_ROOT_HANDLER], "level": env("LOG_LEVEL").upper()} # Sentry if env.str("SENTRY_DSN", ""): From b16ef8eb2e0a5bef5df9ff6f9e7c9e3efefc3a08 Mon Sep 17 00:00:00 2001 From: "mark.j0hnst0n" Date: Thu, 20 Jun 2024 19:55:18 +0100 Subject: [PATCH 12/73] refactor --- api/conf/settings.py | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/api/conf/settings.py b/api/conf/settings.py index 87ccd83e19..0c1a1ff265 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -377,39 +377,27 @@ def _build_redis_url(base_url, db_number, **query_args): ENVIRONMENT = env.str("ENV", "") if ENVIRONMENT == "local": - LOGGING_ROOT_HANDLER = "stdout" LOGGING["formatters"] = { "simple": {"format": "{asctime} {levelname} {message}", "style": "{"}, } - LOGGING["handlers"] = { - "stdout": {"class": "logging.StreamHandler", "formatter": "simple"}, - "sentry": {"class": "sentry_sdk.integrations.logging.EventHandler"}, - } - LOGGING["root"] = {"handlers": [LOGGING_ROOT_HANDLER], "level": env("LOG_LEVEL").upper()} + LOGGING["handlers"]["stdout"] = {"class": "logging.StreamHandler", "formatter": "simple"} + LOGGING["root"] = {"handlers": ["stdout"], "level": env("LOG_LEVEL").upper()} elif VCAP_SERVICES: - LOGGING_ROOT_HANDLER = "ecs" LOGGING["formatters"] = { "ecs_formatter": {"()": ECSFormatter}, } - LOGGING["handlers"] = { - "ecs": {"class": "logging.StreamHandler", "formatter": "ecs_formatter"}, - "sentry": {"class": "sentry_sdk.integrations.logging.EventHandler"}, - } - LOGGING["root"] = {"handlers": [LOGGING_ROOT_HANDLER], "level": env("LOG_LEVEL").upper()} + LOGGING["handlers"]["ecs"] = {"class": "logging.StreamHandler", "formatter": "ecs_formatter"} + LOGGING["root"] = {"handlers": ["ecs"], "level": env("LOG_LEVEL").upper()} else: - LOGGING_ROOT_HANDLER = "asim" LOGGING["formatters"] = { "asim_formatter": { "()": ASIMFormatter, }, } - LOGGING["handlers"] = { - "asim": {"class": "logging.StreamHandler", "formatter": "asim_formatter"}, - "sentry": {"class": "sentry_sdk.integrations.logging.EventHandler"}, - } - LOGGING["root"] = {"handlers": [LOGGING_ROOT_HANDLER], "level": env("LOG_LEVEL").upper()} + LOGGING["handlers"]["asim"] = {"class": "logging.StreamHandler", "formatter": "asim_formatter"} + LOGGING["root"] = {"handlers": ["asim"], "level": env("LOG_LEVEL").upper()} # Sentry if env.str("SENTRY_DSN", ""): From d1d9d0920e50e87b9254d1b3392ae2ebffbb7bfb Mon Sep 17 00:00:00 2001 From: "mark.j0hnst0n" Date: Thu, 20 Jun 2024 21:11:34 +0100 Subject: [PATCH 13/73] dbt_copilot added --- Pipfile | 1 + Pipfile.lock | 382 +++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 372 insertions(+), 11 deletions(-) diff --git a/Pipfile b/Pipfile index cf59bbf14b..f64868c9cb 100644 --- a/Pipfile +++ b/Pipfile @@ -74,6 +74,7 @@ database-sanitizer = ">=1.1.0" django-reversion = ">=5.0.12" psycopg = "~=3.1.18" django-log-formatter-asim = "~=0.0.5" +dbt-copilot-python = "~=0.2.1" [requires] python_version = "3.9" diff --git a/Pipfile.lock b/Pipfile.lock index 47c4d5dfdb..4f5dc33db1 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "5e8956feac774b3af51326cc1b52d81eef45978cc2d37e50f611cfa0c72f116b" + "sha256": "792919a16ef11b6fadff4d71b4f1b37f6b05115c9f69b01b4d5fed61dab00add" }, "pipfile-spec": 6, "requires": { @@ -56,6 +56,14 @@ ], "version": "==0.2.0" }, + "backoff": { + "hashes": [ + "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba", + "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8" + ], + "markers": "python_version >= '3.7'", + "version": "==2.2.1" + }, "billiard": { "hashes": [ "sha256:07aa978b308f334ff8282bd4a746e681b3513db5c9a514cbdd810cbbdc19714d", @@ -370,6 +378,15 @@ "index": "pypi", "version": "==0.6" }, + "dbt-copilot-python": { + "hashes": [ + "sha256:8330011cec44c2e6e30f48b3bdb346269a1704eb35ca19f9666509d8b9d39844", + "sha256:97fa5429ff296e09821b17395f6f6431091a4fa45f21177b19b4a478df49827f" + ], + "index": "pypi", + "markers": "python_version >= '3.9' and python_version < '4.0'", + "version": "==0.2.1" + }, "decorator": { "hashes": [ "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330", @@ -387,6 +404,14 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==0.6.0" }, + "deprecated": { + "hashes": [ + "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c", + "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.2.14" + }, "distlib": { "hashes": [ "sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784", @@ -670,11 +695,11 @@ }, "filelock": { "hashes": [ - "sha256:58a2549afdf9e02e10720eaa4d4470f56386d7a6f72edd7d0596337af8ed7ad8", - "sha256:71b3102950e91dfc1bb4209b64be4dc8854f40e5f534428d8684f953ac847fac" + "sha256:0151273e5b5d6cf753a61ec83b3a9b7d8821c39ae9af9d7ecf2f9e2f17404103", + "sha256:e1199bf5194a2277273dacd50269f0d87d0682088a3c561c15674ea9005d8635" ], "markers": "python_version >= '3.8'", - "version": "==3.15.1" + "version": "==3.15.3" }, "future": { "hashes": [ @@ -731,6 +756,14 @@ "markers": "python_version >= '3.8'", "version": "==23.9.1" }, + "googleapis-common-protos": { + "hashes": [ + "sha256:0e1c2cdfcbc354b76e4a211a35ea35d6926a835cba1377073c4861db904a1877", + "sha256:c6442f7a0a6b2a80369457d79e6672bb7dcbaab88e0848302497e3ec80780a6a" + ], + "markers": "python_version >= '3.7'", + "version": "==1.63.1" + }, "gprof2dot": { "hashes": [ "sha256:45b14ad7ce64e299c8f526881007b9eb2c6b75505d5613e96e66ee4d5ab33696", @@ -803,6 +836,58 @@ "markers": "platform_python_implementation == 'CPython' and python_version < '3.11'", "version": "==3.0.3" }, + "grpcio": { + "hashes": [ + "sha256:03b43d0ccf99c557ec671c7dede64f023c7da9bb632ac65dbc57f166e4970040", + "sha256:0a12ddb1678ebc6a84ec6b0487feac020ee2b1659cbe69b80f06dbffdb249122", + "sha256:0a2813093ddb27418a4c99f9b1c223fab0b053157176a64cc9db0f4557b69bd9", + "sha256:0cc79c982ccb2feec8aad0e8fb0d168bcbca85bc77b080d0d3c5f2f15c24ea8f", + "sha256:1257b76748612aca0f89beec7fa0615727fd6f2a1ad580a9638816a4b2eb18fd", + "sha256:1262402af5a511c245c3ae918167eca57342c72320dffae5d9b51840c4b2f86d", + "sha256:19264fc964576ddb065368cae953f8d0514ecc6cb3da8903766d9fb9d4554c33", + "sha256:198908f9b22e2672a998870355e226a725aeab327ac4e6ff3a1399792ece4762", + "sha256:1de403fc1305fd96cfa75e83be3dee8538f2413a6b1685b8452301c7ba33c294", + "sha256:20405cb8b13fd779135df23fabadc53b86522d0f1cba8cca0e87968587f50650", + "sha256:2981c7365a9353f9b5c864595c510c983251b1ab403e05b1ccc70a3d9541a73b", + "sha256:2c3c1b90ab93fed424e454e93c0ed0b9d552bdf1b0929712b094f5ecfe7a23ad", + "sha256:39b9d0acaa8d835a6566c640f48b50054f422d03e77e49716d4c4e8e279665a1", + "sha256:3b64ae304c175671efdaa7ec9ae2cc36996b681eb63ca39c464958396697daff", + "sha256:4657d24c8063e6095f850b68f2d1ba3b39f2b287a38242dcabc166453e950c59", + "sha256:4d6dab6124225496010bd22690f2d9bd35c7cbb267b3f14e7a3eb05c911325d4", + "sha256:55260032b95c49bee69a423c2f5365baa9369d2f7d233e933564d8a47b893027", + "sha256:55697ecec192bc3f2f3cc13a295ab670f51de29884ca9ae6cd6247df55df2502", + "sha256:5841dd1f284bd1b3d8a6eca3a7f062b06f1eec09b184397e1d1d43447e89a7ae", + "sha256:58b1041e7c870bb30ee41d3090cbd6f0851f30ae4eb68228955d973d3efa2e61", + "sha256:5e42634a989c3aa6049f132266faf6b949ec2a6f7d302dbb5c15395b77d757eb", + "sha256:5e56462b05a6f860b72f0fa50dca06d5b26543a4e88d0396259a07dc30f4e5aa", + "sha256:5f8b75f64d5d324c565b263c67dbe4f0af595635bbdd93bb1a88189fc62ed2e5", + "sha256:62b4e6eb7bf901719fce0ca83e3ed474ae5022bb3827b0a501e056458c51c0a1", + "sha256:6503b64c8b2dfad299749cad1b595c650c91e5b2c8a1b775380fcf8d2cbba1e9", + "sha256:6c024ffc22d6dc59000faf8ad781696d81e8e38f4078cb0f2630b4a3cf231a90", + "sha256:73819689c169417a4f978e562d24f2def2be75739c4bed1992435d007819da1b", + "sha256:75dbbf415026d2862192fe1b28d71f209e2fd87079d98470db90bebe57b33179", + "sha256:8caee47e970b92b3dd948371230fcceb80d3f2277b3bf7fbd7c0564e7d39068e", + "sha256:8d51dd1c59d5fa0f34266b80a3805ec29a1f26425c2a54736133f6d87fc4968a", + "sha256:940e3ec884520155f68a3b712d045e077d61c520a195d1a5932c531f11883489", + "sha256:a011ac6c03cfe162ff2b727bcb530567826cec85eb8d4ad2bfb4bd023287a52d", + "sha256:a3a035c37ce7565b8f4f35ff683a4db34d24e53dc487e47438e434eb3f701b2a", + "sha256:a5e771d0252e871ce194d0fdcafd13971f1aae0ddacc5f25615030d5df55c3a2", + "sha256:ac15b6c2c80a4d1338b04d42a02d376a53395ddf0ec9ab157cbaf44191f3ffdd", + "sha256:b1a82e0b9b3022799c336e1fc0f6210adc019ae84efb7321d668129d28ee1efb", + "sha256:bac71b4b28bc9af61efcdc7630b166440bbfbaa80940c9a697271b5e1dabbc61", + "sha256:bbc5b1d78a7822b0a84c6f8917faa986c1a744e65d762ef6d8be9d75677af2ca", + "sha256:c1a786ac592b47573a5bb7e35665c08064a5d77ab88a076eec11f8ae86b3e3f6", + "sha256:c84ad903d0d94311a2b7eea608da163dace97c5fe9412ea311e72c3684925602", + "sha256:d4d29cc612e1332237877dfa7fe687157973aab1d63bd0f84cf06692f04c0367", + "sha256:e3d9f8d1221baa0ced7ec7322a981e28deb23749c76eeeb3d33e18b72935ab62", + "sha256:e7cd5c1325f6808b8ae31657d281aadb2a51ac11ab081ae335f4f7fc44c1721d", + "sha256:ed6091fa0adcc7e4ff944090cf203a52da35c37a130efa564ded02b7aff63bcd", + "sha256:ee73a2f5ca4ba44fa33b4d7d2c71e2c8a9e9f78d53f6507ad68e7d2ad5f64a22", + "sha256:f10193c69fc9d3d726e83bbf0f3d316f1847c3071c8c93d8090cf5f326b14309" + ], + "markers": "python_version >= '3.8'", + "version": "==1.64.1" + }, "gunicorn": { "hashes": [ "sha256:350679f91b24062c86e386e198a15438d53a7a8207235a78ba1b53df4c4378d9", @@ -838,11 +923,11 @@ }, "importlib-metadata": { "hashes": [ - "sha256:30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570", - "sha256:b78938b926ee8d5f020fc4772d487045805a55ddbad2ecf21c6d60938dc7fcd2" + "sha256:1231cf92d825c9e03cfc4da076a16de6422c863558229ea0b22b675657463443", + "sha256:f0afba6205ad8f8947c7d338b5342d5db2afbfd82f9cbef7879a9539cc12eb9b" ], "markers": "python_version < '3.10'", - "version": "==7.1.0" + "version": "==6.11.0" }, "ipython": { "hashes": [ @@ -964,6 +1049,118 @@ "markers": "python_version >= '3.6'", "version": "==3.0.6" }, + "opentelemetry-api": { + "hashes": [ + "sha256:15ae4ca925ecf9cfdfb7a709250846fbb08072260fca08ade78056c502b86bed", + "sha256:43621514301a7e9f5d06dd8013a1b450f30c2e9372b8e30aaeb4562abf2ce034" + ], + "markers": "python_version >= '3.7'", + "version": "==1.22.0" + }, + "opentelemetry-distro": { + "hashes": [ + "sha256:a33f4a5f40f51dbb9f1d21b426d128312f9e2a12fe04da31a9017da6c20c958e", + "sha256:ab4876203d06d40c77cffe1e79c859cff6b471725a846076946cc94df1a863a9" + ], + "markers": "python_version >= '3.7'", + "version": "==0.43b0" + }, + "opentelemetry-exporter-otlp": { + "hashes": [ + "sha256:309a7d4dc67602801f15818e110ce452e78989886aaab5d37e7cf7f55f1d3d27", + "sha256:cb03a1cbf300e12b47690858be13dd26fe2f60b2610204959f3497cd6645e3a1" + ], + "markers": "python_version >= '3.7'", + "version": "==1.22.0" + }, + "opentelemetry-exporter-otlp-proto-common": { + "hashes": [ + "sha256:3f2538bec5312587f8676c332b3747f54c89fe6364803a807e217af4603201fa", + "sha256:71ae2f81bc6d6fe408d06388826edc8933759b2ca3a97d24054507dc7cfce52d" + ], + "markers": "python_version >= '3.7'", + "version": "==1.22.0" + }, + "opentelemetry-exporter-otlp-proto-grpc": { + "hashes": [ + "sha256:1e0e5aa4bbabc74942f06f268deffd94851d12a8dc30b02527472ef1729fe5b1", + "sha256:b5bcadc129272004316a455e9081216d3380c1fc2231a928ea6a70aa90e173fb" + ], + "markers": "python_version >= '3.7'", + "version": "==1.22.0" + }, + "opentelemetry-exporter-otlp-proto-http": { + "hashes": [ + "sha256:79ed108981ec68d5f7985355bca32003c2f3a5be1534a96d62d5861b758a82f4", + "sha256:e002e842190af45b91dc55a97789d0b98e4308c88d886b16049ee90e17a4d396" + ], + "markers": "python_version >= '3.7'", + "version": "==1.22.0" + }, + "opentelemetry-instrumentation": { + "hashes": [ + "sha256:0ff1334d7e359e27640e9d420024efeb73eacae464309c2e14ede7ba6c93967e", + "sha256:c3755da6c4be8033be0216d0501e11f4832690f4e2eca5a3576fbf113498f0f6" + ], + "markers": "python_version >= '3.7'", + "version": "==0.43b0" + }, + "opentelemetry-instrumentation-wsgi": { + "hashes": [ + "sha256:0b7511469daa29a6e75b9cc54b4d01a9bb46aa1f964471dc3ee3f06ff39f94b2", + "sha256:3a1cf045f7ccf04987a89cdd49eda93e9195de4c8b73be228a9e565ec3ab453c" + ], + "markers": "python_version >= '3.7'", + "version": "==0.43b0" + }, + "opentelemetry-propagator-aws-xray": { + "hashes": [ + "sha256:49267a1d72b3f04880ac75e24f9ef38fe323e2f3156c4531e0e00c71c0829c0f", + "sha256:6e8be667bbcf17c3d81d70b2a7cdec0b11257ff64d3829ffe75b810ba1b49f86" + ], + "markers": "python_version >= '3.6'", + "version": "==1.0.1" + }, + "opentelemetry-proto": { + "hashes": [ + "sha256:9ec29169286029f17ca34ec1f3455802ffb90131642d2f545ece9a63e8f69003", + "sha256:ce7188d22c75b6d0fe53e7fb58501613d0feade5139538e79dedd9420610fa0c" + ], + "markers": "python_version >= '3.7'", + "version": "==1.22.0" + }, + "opentelemetry-sdk": { + "hashes": [ + "sha256:45267ac1f38a431fc2eb5d6e0c0d83afc0b78de57ac345488aa58c28c17991d0", + "sha256:a730555713d7c8931657612a88a141e3a4fe6eb5523d9e2d5a8b1e673d76efa6" + ], + "markers": "python_version >= '3.7'", + "version": "==1.22.0" + }, + "opentelemetry-sdk-extension-aws": { + "hashes": [ + "sha256:dd7cf6fc0e7c8070dbe179348f2f194ca4555601b60efb7264d82fc8df53f4ba", + "sha256:f964b0598793ded268d3329c33829fad33f63a8d9299fe51bf3a743e81fd7c67" + ], + "markers": "python_version >= '3.6'", + "version": "==2.0.1" + }, + "opentelemetry-semantic-conventions": { + "hashes": [ + "sha256:291284d7c1bf15fdaddf309b3bd6d3b7ce12a253cec6d27144439819a15d8445", + "sha256:b9576fb890df479626fa624e88dde42d3d60b8b6c8ae1152ad157a8b97358635" + ], + "markers": "python_version >= '3.7'", + "version": "==0.43b0" + }, + "opentelemetry-util-http": { + "hashes": [ + "sha256:3ff6ab361dbe99fc81200d625603c0fb890c055c6e416a3e6d661ddf47a6c7f7", + "sha256:f25a820784b030f6cb86b3d76e5676c769b75ed3f55a210bcdae0a5e175ebadb" + ], + "markers": "python_version >= '3.7'", + "version": "==0.43b0" + }, "packaging": { "hashes": [ "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002", @@ -1102,6 +1299,23 @@ "markers": "python_full_version >= '3.7.0'", "version": "==3.0.47" }, + "protobuf": { + "hashes": [ + "sha256:19b270aeaa0099f16d3ca02628546b8baefe2955bbe23224aaf856134eccf1e4", + "sha256:209ba4cc916bab46f64e56b85b090607a676f66b473e6b762e6f1d9d591eb2e8", + "sha256:25b5d0b42fd000320bd7830b349e3b696435f3b329810427a6bcce6a5492cc5c", + "sha256:7c8daa26095f82482307bc717364e7c13f4f1c99659be82890dcfc215194554d", + "sha256:c053062984e61144385022e53678fbded7aea14ebb3e0305ae3592fb219ccfa4", + "sha256:d4198877797a83cbfe9bffa3803602bbe1625dc30d8a097365dbc762e5790faa", + "sha256:e3c97a1555fd6388f857770ff8b9703083de6bf1f9274a002a332d65fbb56c8c", + "sha256:e7cb0ae90dd83727f0c0718634ed56837bfeeee29a5f82a7514c03ee1364c019", + "sha256:f0700d54bcf45424477e46a9f0944155b46fb0639d69728739c0e47bab83f2b9", + "sha256:f1279ab38ecbfae7e456a108c5c0681e4956d5b1090027c1de0f934dfdb4b35c", + "sha256:f4f118245c4a087776e0a8408be33cf09f6c547442c00395fbfb116fac2f8ac2" + ], + "markers": "python_version >= '3.8'", + "version": "==4.25.3" + }, "psycopg": { "hashes": [ "sha256:92d7b78ad82426cdcf1a0440678209faa890c6e1721361c2f8901f0dccd62961", @@ -1375,7 +1589,7 @@ "sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3", "sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429" ], - "markers": "python_version >= '3.6'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", "version": "==1.26.19" }, "vine": { @@ -1426,6 +1640,82 @@ "markers": "python_version >= '3.5' and python_version < '4'", "version": "==5.3.0" }, + "wrapt": { + "hashes": [ + "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc", + "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81", + "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09", + "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e", + "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca", + "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0", + "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb", + "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487", + "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40", + "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c", + "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060", + "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202", + "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41", + "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9", + "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b", + "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664", + "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d", + "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362", + "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00", + "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc", + "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1", + "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267", + "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956", + "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966", + "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1", + "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228", + "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72", + "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d", + "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292", + "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0", + "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0", + "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36", + "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c", + "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5", + "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f", + "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73", + "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b", + "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2", + "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593", + "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39", + "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389", + "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf", + "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf", + "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89", + "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c", + "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c", + "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f", + "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440", + "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465", + "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136", + "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b", + "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8", + "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3", + "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8", + "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6", + "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e", + "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f", + "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c", + "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e", + "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8", + "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2", + "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020", + "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35", + "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d", + "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3", + "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537", + "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809", + "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d", + "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a", + "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4" + ], + "markers": "python_version >= '3.6'", + "version": "==1.16.0" + }, "xmlschema": { "hashes": [ "sha256:1c4515dd16b5b556dac27cae70bdb3c863e1448e9d58896326ff468ff55e4246", @@ -2545,7 +2835,7 @@ "sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3", "sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429" ], - "markers": "python_version >= '3.6'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", "version": "==1.26.19" }, "watchdog": { @@ -2606,9 +2896,79 @@ }, "wrapt": { "hashes": [ - "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" + "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc", + "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81", + "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09", + "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e", + "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca", + "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0", + "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb", + "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487", + "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40", + "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c", + "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060", + "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202", + "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41", + "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9", + "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b", + "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664", + "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d", + "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362", + "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00", + "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc", + "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1", + "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267", + "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956", + "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966", + "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1", + "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228", + "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72", + "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d", + "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292", + "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0", + "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0", + "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36", + "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c", + "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5", + "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f", + "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73", + "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b", + "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2", + "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593", + "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39", + "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389", + "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf", + "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf", + "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89", + "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c", + "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c", + "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f", + "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440", + "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465", + "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136", + "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b", + "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8", + "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3", + "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8", + "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6", + "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e", + "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f", + "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c", + "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e", + "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8", + "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2", + "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020", + "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35", + "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d", + "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3", + "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537", + "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809", + "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d", + "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a", + "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4" ], - "version": "==1.11.2" + "markers": "python_version >= '3.6'", + "version": "==1.16.0" }, "xmltodict": { "hashes": [ From f7368433d5003baa0dc7a976b374a00e1274c74f Mon Sep 17 00:00:00 2001 From: "mark.j0hnst0n" Date: Thu, 20 Jun 2024 21:59:04 +0100 Subject: [PATCH 14/73] revert --- api/conf/settings.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/api/conf/settings.py b/api/conf/settings.py index 0c1a1ff265..afd7ed205a 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -9,6 +9,7 @@ from sentry_sdk.integrations.django import DjangoIntegration from django_log_formatter_ecs import ECSFormatter from django_log_formatter_asim import ASIMFormatter +from dbt_copilot_python.utility import is_copilot from django.urls import reverse_lazy @@ -383,14 +384,7 @@ def _build_redis_url(base_url, db_number, **query_args): LOGGING["handlers"]["stdout"] = {"class": "logging.StreamHandler", "formatter": "simple"} LOGGING["root"] = {"handlers": ["stdout"], "level": env("LOG_LEVEL").upper()} -elif VCAP_SERVICES: - LOGGING["formatters"] = { - "ecs_formatter": {"()": ECSFormatter}, - } - LOGGING["handlers"]["ecs"] = {"class": "logging.StreamHandler", "formatter": "ecs_formatter"} - LOGGING["root"] = {"handlers": ["ecs"], "level": env("LOG_LEVEL").upper()} - -else: +if is_copilot(): LOGGING["formatters"] = { "asim_formatter": { "()": ASIMFormatter, @@ -398,6 +392,12 @@ def _build_redis_url(base_url, db_number, **query_args): } LOGGING["handlers"]["asim"] = {"class": "logging.StreamHandler", "formatter": "asim_formatter"} LOGGING["root"] = {"handlers": ["asim"], "level": env("LOG_LEVEL").upper()} +else: + LOGGING["formatters"] = { + "ecs_formatter": {"()": ECSFormatter}, + } + LOGGING["handlers"]["ecs"] = {"class": "logging.StreamHandler", "formatter": "ecs_formatter"} + LOGGING["root"] = {"handlers": ["ecs"], "level": env("LOG_LEVEL").upper()} # Sentry if env.str("SENTRY_DSN", ""): From 57fc0d4913561a8574f82f6ebf1fecd9760a871c Mon Sep 17 00:00:00 2001 From: "mark.j0hnst0n" Date: Thu, 20 Jun 2024 23:01:54 +0100 Subject: [PATCH 15/73] reformat --- api/conf/settings.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/api/conf/settings.py b/api/conf/settings.py index afd7ed205a..ba9e14e364 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -384,7 +384,14 @@ def _build_redis_url(base_url, db_number, **query_args): LOGGING["handlers"]["stdout"] = {"class": "logging.StreamHandler", "formatter": "simple"} LOGGING["root"] = {"handlers": ["stdout"], "level": env("LOG_LEVEL").upper()} -if is_copilot(): +elif not is_copilot(): + LOGGING["formatters"] = { + "ecs_formatter": {"()": ECSFormatter}, + } + LOGGING["handlers"]["ecs"] = {"class": "logging.StreamHandler", "formatter": "ecs_formatter"} + LOGGING["root"] = {"handlers": ["ecs"], "level": env("LOG_LEVEL").upper()} + +else: LOGGING["formatters"] = { "asim_formatter": { "()": ASIMFormatter, @@ -392,12 +399,7 @@ def _build_redis_url(base_url, db_number, **query_args): } LOGGING["handlers"]["asim"] = {"class": "logging.StreamHandler", "formatter": "asim_formatter"} LOGGING["root"] = {"handlers": ["asim"], "level": env("LOG_LEVEL").upper()} -else: - LOGGING["formatters"] = { - "ecs_formatter": {"()": ECSFormatter}, - } - LOGGING["handlers"]["ecs"] = {"class": "logging.StreamHandler", "formatter": "ecs_formatter"} - LOGGING["root"] = {"handlers": ["ecs"], "level": env("LOG_LEVEL").upper()} + # Sentry if env.str("SENTRY_DSN", ""): From 3cd3f866dec5a0751b52ccaea035dc903083c43e Mon Sep 17 00:00:00 2001 From: "mark.j0hnst0n" Date: Thu, 20 Jun 2024 23:10:34 +0100 Subject: [PATCH 16/73] revert --- api/conf/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/conf/settings.py b/api/conf/settings.py index ba9e14e364..5c0a03c834 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -384,14 +384,14 @@ def _build_redis_url(base_url, db_number, **query_args): LOGGING["handlers"]["stdout"] = {"class": "logging.StreamHandler", "formatter": "simple"} LOGGING["root"] = {"handlers": ["stdout"], "level": env("LOG_LEVEL").upper()} -elif not is_copilot(): +elif VCAP_SERVICES: LOGGING["formatters"] = { "ecs_formatter": {"()": ECSFormatter}, } LOGGING["handlers"]["ecs"] = {"class": "logging.StreamHandler", "formatter": "ecs_formatter"} LOGGING["root"] = {"handlers": ["ecs"], "level": env("LOG_LEVEL").upper()} -else: +elif is_copilot(): LOGGING["formatters"] = { "asim_formatter": { "()": ASIMFormatter, From 5e42d80c1796171cec67071c2ff403353699e670 Mon Sep 17 00:00:00 2001 From: "mark.j0hnst0n" Date: Thu, 20 Jun 2024 23:14:55 +0100 Subject: [PATCH 17/73] Update settings.py --- api/conf/settings.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/conf/settings.py b/api/conf/settings.py index 5c0a03c834..ea398613c1 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -9,7 +9,6 @@ from sentry_sdk.integrations.django import DjangoIntegration from django_log_formatter_ecs import ECSFormatter from django_log_formatter_asim import ASIMFormatter -from dbt_copilot_python.utility import is_copilot from django.urls import reverse_lazy @@ -391,7 +390,7 @@ def _build_redis_url(base_url, db_number, **query_args): LOGGING["handlers"]["ecs"] = {"class": "logging.StreamHandler", "formatter": "ecs_formatter"} LOGGING["root"] = {"handlers": ["ecs"], "level": env("LOG_LEVEL").upper()} -elif is_copilot(): +else: LOGGING["formatters"] = { "asim_formatter": { "()": ASIMFormatter, From 5db56ac69516ebb36287bb29d50f81fd535b56ac Mon Sep 17 00:00:00 2001 From: "mark.j0hnst0n" Date: Thu, 20 Jun 2024 23:22:14 +0100 Subject: [PATCH 18/73] Update settings.py --- api/conf/settings.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/api/conf/settings.py b/api/conf/settings.py index ea398613c1..38d4728eec 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -9,6 +9,8 @@ from sentry_sdk.integrations.django import DjangoIntegration from django_log_formatter_ecs import ECSFormatter from django_log_formatter_asim import ASIMFormatter +from dbt_copilot_python.utility import is_copilot + from django.urls import reverse_lazy @@ -383,6 +385,7 @@ def _build_redis_url(base_url, db_number, **query_args): LOGGING["handlers"]["stdout"] = {"class": "logging.StreamHandler", "formatter": "simple"} LOGGING["root"] = {"handlers": ["stdout"], "level": env("LOG_LEVEL").upper()} + elif VCAP_SERVICES: LOGGING["formatters"] = { "ecs_formatter": {"()": ECSFormatter}, @@ -390,7 +393,7 @@ def _build_redis_url(base_url, db_number, **query_args): LOGGING["handlers"]["ecs"] = {"class": "logging.StreamHandler", "formatter": "ecs_formatter"} LOGGING["root"] = {"handlers": ["ecs"], "level": env("LOG_LEVEL").upper()} -else: +elif is_copilot(): LOGGING["formatters"] = { "asim_formatter": { "()": ASIMFormatter, From cd5500038851255307503a64c4fa47bdcc1346d0 Mon Sep 17 00:00:00 2001 From: "mark.j0hnst0n" Date: Thu, 20 Jun 2024 23:24:10 +0100 Subject: [PATCH 19/73] revert --- api/conf/settings.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/api/conf/settings.py b/api/conf/settings.py index 38d4728eec..9b0f834356 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -385,14 +385,6 @@ def _build_redis_url(base_url, db_number, **query_args): LOGGING["handlers"]["stdout"] = {"class": "logging.StreamHandler", "formatter": "simple"} LOGGING["root"] = {"handlers": ["stdout"], "level": env("LOG_LEVEL").upper()} - -elif VCAP_SERVICES: - LOGGING["formatters"] = { - "ecs_formatter": {"()": ECSFormatter}, - } - LOGGING["handlers"]["ecs"] = {"class": "logging.StreamHandler", "formatter": "ecs_formatter"} - LOGGING["root"] = {"handlers": ["ecs"], "level": env("LOG_LEVEL").upper()} - elif is_copilot(): LOGGING["formatters"] = { "asim_formatter": { @@ -402,6 +394,13 @@ def _build_redis_url(base_url, db_number, **query_args): LOGGING["handlers"]["asim"] = {"class": "logging.StreamHandler", "formatter": "asim_formatter"} LOGGING["root"] = {"handlers": ["asim"], "level": env("LOG_LEVEL").upper()} +else: + LOGGING["formatters"] = { + "ecs_formatter": {"()": ECSFormatter}, + } + LOGGING["handlers"]["ecs"] = {"class": "logging.StreamHandler", "formatter": "ecs_formatter"} + LOGGING["root"] = {"handlers": ["ecs"], "level": env("LOG_LEVEL").upper()} + # Sentry if env.str("SENTRY_DSN", ""): From 9f7d75b83bc7765f2bb30458cb885646f9bc9a37 Mon Sep 17 00:00:00 2001 From: "mark.j0hnst0n" Date: Thu, 20 Jun 2024 23:59:40 +0100 Subject: [PATCH 20/73] elif not co_pilot used --- api/conf/settings.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/api/conf/settings.py b/api/conf/settings.py index 9b0f834356..16619f34f8 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -385,7 +385,14 @@ def _build_redis_url(base_url, db_number, **query_args): LOGGING["handlers"]["stdout"] = {"class": "logging.StreamHandler", "formatter": "simple"} LOGGING["root"] = {"handlers": ["stdout"], "level": env("LOG_LEVEL").upper()} -elif is_copilot(): +elif not is_copilot(): + LOGGING["formatters"] = { + "ecs_formatter": {"()": ECSFormatter}, + } + LOGGING["handlers"]["ecs"] = {"class": "logging.StreamHandler", "formatter": "ecs_formatter"} + LOGGING["root"] = {"handlers": ["ecs"], "level": env("LOG_LEVEL").upper()} + +else: LOGGING["formatters"] = { "asim_formatter": { "()": ASIMFormatter, @@ -394,13 +401,6 @@ def _build_redis_url(base_url, db_number, **query_args): LOGGING["handlers"]["asim"] = {"class": "logging.StreamHandler", "formatter": "asim_formatter"} LOGGING["root"] = {"handlers": ["asim"], "level": env("LOG_LEVEL").upper()} -else: - LOGGING["formatters"] = { - "ecs_formatter": {"()": ECSFormatter}, - } - LOGGING["handlers"]["ecs"] = {"class": "logging.StreamHandler", "formatter": "ecs_formatter"} - LOGGING["root"] = {"handlers": ["ecs"], "level": env("LOG_LEVEL").upper()} - # Sentry if env.str("SENTRY_DSN", ""): From 2a1e94ccec10fe281300097d248ef12f010e83e9 Mon Sep 17 00:00:00 2001 From: "mark.j0hnst0n" Date: Fri, 21 Jun 2024 10:34:12 +0100 Subject: [PATCH 21/73] used update method on logging dict --- api/conf/settings.py | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/api/conf/settings.py b/api/conf/settings.py index 16619f34f8..1349cf40f5 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -371,6 +371,7 @@ def _build_redis_url(base_url, db_number, **query_args): "handlers": { "sentry": {"class": "sentry_sdk.integrations.logging.EventHandler"}, }, + "root": {"handlers": ["sentry"], "level": env("LOG_LEVEL").upper()}, "loggers": { DENIAL_REASONS_DELETION_LOGGER: {"handlers": ["sentry"], "level": logging.WARNING}, }, @@ -379,27 +380,20 @@ def _build_redis_url(base_url, db_number, **query_args): ENVIRONMENT = env.str("ENV", "") if ENVIRONMENT == "local": - LOGGING["formatters"] = { - "simple": {"format": "{asctime} {levelname} {message}", "style": "{"}, - } - LOGGING["handlers"]["stdout"] = {"class": "logging.StreamHandler", "formatter": "simple"} - LOGGING["root"] = {"handlers": ["stdout"], "level": env("LOG_LEVEL").upper()} + LOGGING.update({"formatters": {"simple": {"format": "{asctime} {levelname} {message}", "style": "{"}}}) + LOGGING["handlers"].update({"stdout": {"class": "logging.StreamHandler", "formatter": "simple"}}) + LOGGING.update({"root": {"handlers": ["stdout"], "level": env("LOG_LEVEL").upper()}}) + elif not is_copilot(): - LOGGING["formatters"] = { - "ecs_formatter": {"()": ECSFormatter}, - } - LOGGING["handlers"]["ecs"] = {"class": "logging.StreamHandler", "formatter": "ecs_formatter"} - LOGGING["root"] = {"handlers": ["ecs"], "level": env("LOG_LEVEL").upper()} + LOGGING.update({"formatters": {"ecs_formatter": {"()": ECSFormatter}}}) + LOGGING["handlers"].update({"ecs": {"class": "logging.StreamHandler", "formatter": "ecs_formatter"}}) + LOGGING.update({"root": {"handlers": ["ecs"], "level": env("LOG_LEVEL").upper()}}) else: - LOGGING["formatters"] = { - "asim_formatter": { - "()": ASIMFormatter, - }, - } - LOGGING["handlers"]["asim"] = {"class": "logging.StreamHandler", "formatter": "asim_formatter"} - LOGGING["root"] = {"handlers": ["asim"], "level": env("LOG_LEVEL").upper()} + LOGGING.update({"formatters": {"asim_formatter": {"()": ASIMFormatter}}}) + LOGGING["handlers"].update({"asim": {"class": "logging.StreamHandler", "formatter": "asim_formatter"}}) + LOGGING.update({"root": {"handlers": ["asim"], "level": env("LOG_LEVEL").upper()}}) # Sentry From ea7421c5a63d46e3f78e94cf30c096b850c49eb8 Mon Sep 17 00:00:00 2001 From: "mark.j0hnst0n" Date: Fri, 21 Jun 2024 10:35:05 +0100 Subject: [PATCH 22/73] remove unnecessary root ref --- api/conf/settings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/api/conf/settings.py b/api/conf/settings.py index 1349cf40f5..7a2bff807e 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -371,7 +371,6 @@ def _build_redis_url(base_url, db_number, **query_args): "handlers": { "sentry": {"class": "sentry_sdk.integrations.logging.EventHandler"}, }, - "root": {"handlers": ["sentry"], "level": env("LOG_LEVEL").upper()}, "loggers": { DENIAL_REASONS_DELETION_LOGGER: {"handlers": ["sentry"], "level": logging.WARNING}, }, From bba533a5ba48e1204e411b4dd97c65634d11ebea Mon Sep 17 00:00:00 2001 From: "mark.j0hnst0n" Date: Fri, 21 Jun 2024 13:01:16 +0100 Subject: [PATCH 23/73] Update Pipfile.lock --- Pipfile.lock | 380 ++------------------------------------------------- 1 file changed, 10 insertions(+), 370 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index ed6a98ab4a..860ef4c9bf 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -56,18 +56,13 @@ ], "version": "==0.2.0" }, - "backoff": { "backoff": { "hashes": [ "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba", "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8" - "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba", - "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8" ], "markers": "python_version >= '3.7'", "version": "==2.2.1" - "markers": "python_version >= '3.7'", - "version": "==2.2.1" }, "billiard": { "hashes": [ @@ -392,15 +387,6 @@ "markers": "python_version >= '3.9' and python_version < '4.0'", "version": "==0.2.1" }, - "dbt-copilot-python": { - "hashes": [ - "sha256:8330011cec44c2e6e30f48b3bdb346269a1704eb35ca19f9666509d8b9d39844", - "sha256:97fa5429ff296e09821b17395f6f6431091a4fa45f21177b19b4a478df49827f" - ], - "index": "pypi", - "markers": "python_version >= '3.9' and python_version < '4.0'", - "version": "==0.2.1" - }, "decorator": { "hashes": [ "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330", @@ -720,7 +706,7 @@ "sha256:929292d34f5872e70396626ef385ec22355a1fae8ad29e1a734c3e43f9fbc216", "sha256:bd2968309307861edae1458a4f8a4f3598c03be43b97521076aebf5d94c07b05" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", "version": "==1.0.0" }, "gevent": { @@ -778,14 +764,6 @@ "markers": "python_version >= '3.7'", "version": "==1.63.1" }, - "googleapis-common-protos": { - "hashes": [ - "sha256:0e1c2cdfcbc354b76e4a211a35ea35d6926a835cba1377073c4861db904a1877", - "sha256:c6442f7a0a6b2a80369457d79e6672bb7dcbaab88e0848302497e3ec80780a6a" - ], - "markers": "python_version >= '3.7'", - "version": "==1.63.1" - }, "gprof2dot": { "hashes": [ "sha256:45b14ad7ce64e299c8f526881007b9eb2c6b75505d5613e96e66ee4d5ab33696", @@ -855,7 +833,7 @@ "sha256:fd096eb7ffef17c456cfa587523c5f92321ae02427ff955bebe9e3c63bc9f0da", "sha256:fe754d231288e1e64323cfad462fcee8f0288654c10bdf4f603a39ed923bef33" ], - "markers": "python_version < '3.11' and platform_python_implementation == 'CPython'", + "markers": "platform_python_implementation == 'CPython' and python_version < '3.11'", "version": "==3.0.3" }, "grpcio": { @@ -910,58 +888,6 @@ "markers": "python_version >= '3.8'", "version": "==1.64.1" }, - "grpcio": { - "hashes": [ - "sha256:03b43d0ccf99c557ec671c7dede64f023c7da9bb632ac65dbc57f166e4970040", - "sha256:0a12ddb1678ebc6a84ec6b0487feac020ee2b1659cbe69b80f06dbffdb249122", - "sha256:0a2813093ddb27418a4c99f9b1c223fab0b053157176a64cc9db0f4557b69bd9", - "sha256:0cc79c982ccb2feec8aad0e8fb0d168bcbca85bc77b080d0d3c5f2f15c24ea8f", - "sha256:1257b76748612aca0f89beec7fa0615727fd6f2a1ad580a9638816a4b2eb18fd", - "sha256:1262402af5a511c245c3ae918167eca57342c72320dffae5d9b51840c4b2f86d", - "sha256:19264fc964576ddb065368cae953f8d0514ecc6cb3da8903766d9fb9d4554c33", - "sha256:198908f9b22e2672a998870355e226a725aeab327ac4e6ff3a1399792ece4762", - "sha256:1de403fc1305fd96cfa75e83be3dee8538f2413a6b1685b8452301c7ba33c294", - "sha256:20405cb8b13fd779135df23fabadc53b86522d0f1cba8cca0e87968587f50650", - "sha256:2981c7365a9353f9b5c864595c510c983251b1ab403e05b1ccc70a3d9541a73b", - "sha256:2c3c1b90ab93fed424e454e93c0ed0b9d552bdf1b0929712b094f5ecfe7a23ad", - "sha256:39b9d0acaa8d835a6566c640f48b50054f422d03e77e49716d4c4e8e279665a1", - "sha256:3b64ae304c175671efdaa7ec9ae2cc36996b681eb63ca39c464958396697daff", - "sha256:4657d24c8063e6095f850b68f2d1ba3b39f2b287a38242dcabc166453e950c59", - "sha256:4d6dab6124225496010bd22690f2d9bd35c7cbb267b3f14e7a3eb05c911325d4", - "sha256:55260032b95c49bee69a423c2f5365baa9369d2f7d233e933564d8a47b893027", - "sha256:55697ecec192bc3f2f3cc13a295ab670f51de29884ca9ae6cd6247df55df2502", - "sha256:5841dd1f284bd1b3d8a6eca3a7f062b06f1eec09b184397e1d1d43447e89a7ae", - "sha256:58b1041e7c870bb30ee41d3090cbd6f0851f30ae4eb68228955d973d3efa2e61", - "sha256:5e42634a989c3aa6049f132266faf6b949ec2a6f7d302dbb5c15395b77d757eb", - "sha256:5e56462b05a6f860b72f0fa50dca06d5b26543a4e88d0396259a07dc30f4e5aa", - "sha256:5f8b75f64d5d324c565b263c67dbe4f0af595635bbdd93bb1a88189fc62ed2e5", - "sha256:62b4e6eb7bf901719fce0ca83e3ed474ae5022bb3827b0a501e056458c51c0a1", - "sha256:6503b64c8b2dfad299749cad1b595c650c91e5b2c8a1b775380fcf8d2cbba1e9", - "sha256:6c024ffc22d6dc59000faf8ad781696d81e8e38f4078cb0f2630b4a3cf231a90", - "sha256:73819689c169417a4f978e562d24f2def2be75739c4bed1992435d007819da1b", - "sha256:75dbbf415026d2862192fe1b28d71f209e2fd87079d98470db90bebe57b33179", - "sha256:8caee47e970b92b3dd948371230fcceb80d3f2277b3bf7fbd7c0564e7d39068e", - "sha256:8d51dd1c59d5fa0f34266b80a3805ec29a1f26425c2a54736133f6d87fc4968a", - "sha256:940e3ec884520155f68a3b712d045e077d61c520a195d1a5932c531f11883489", - "sha256:a011ac6c03cfe162ff2b727bcb530567826cec85eb8d4ad2bfb4bd023287a52d", - "sha256:a3a035c37ce7565b8f4f35ff683a4db34d24e53dc487e47438e434eb3f701b2a", - "sha256:a5e771d0252e871ce194d0fdcafd13971f1aae0ddacc5f25615030d5df55c3a2", - "sha256:ac15b6c2c80a4d1338b04d42a02d376a53395ddf0ec9ab157cbaf44191f3ffdd", - "sha256:b1a82e0b9b3022799c336e1fc0f6210adc019ae84efb7321d668129d28ee1efb", - "sha256:bac71b4b28bc9af61efcdc7630b166440bbfbaa80940c9a697271b5e1dabbc61", - "sha256:bbc5b1d78a7822b0a84c6f8917faa986c1a744e65d762ef6d8be9d75677af2ca", - "sha256:c1a786ac592b47573a5bb7e35665c08064a5d77ab88a076eec11f8ae86b3e3f6", - "sha256:c84ad903d0d94311a2b7eea608da163dace97c5fe9412ea311e72c3684925602", - "sha256:d4d29cc612e1332237877dfa7fe687157973aab1d63bd0f84cf06692f04c0367", - "sha256:e3d9f8d1221baa0ced7ec7322a981e28deb23749c76eeeb3d33e18b72935ab62", - "sha256:e7cd5c1325f6808b8ae31657d281aadb2a51ac11ab081ae335f4f7fc44c1721d", - "sha256:ed6091fa0adcc7e4ff944090cf203a52da35c37a130efa564ded02b7aff63bcd", - "sha256:ee73a2f5ca4ba44fa33b4d7d2c71e2c8a9e9f78d53f6507ad68e7d2ad5f64a22", - "sha256:f10193c69fc9d3d726e83bbf0f3d316f1847c3071c8c93d8090cf5f326b14309" - ], - "markers": "python_version >= '3.8'", - "version": "==1.64.1" - }, "gunicorn": { "hashes": [ "sha256:350679f91b24062c86e386e198a15438d53a7a8207235a78ba1b53df4c4378d9", @@ -999,12 +925,9 @@ "hashes": [ "sha256:1231cf92d825c9e03cfc4da076a16de6422c863558229ea0b22b675657463443", "sha256:f0afba6205ad8f8947c7d338b5342d5db2afbfd82f9cbef7879a9539cc12eb9b" - "sha256:1231cf92d825c9e03cfc4da076a16de6422c863558229ea0b22b675657463443", - "sha256:f0afba6205ad8f8947c7d338b5342d5db2afbfd82f9cbef7879a9539cc12eb9b" ], "markers": "python_version < '3.10'", "version": "==6.11.0" - "version": "==6.11.0" }, "ipython": { "hashes": [ @@ -1238,118 +1161,6 @@ "markers": "python_version >= '3.7'", "version": "==0.43b0" }, - "opentelemetry-api": { - "hashes": [ - "sha256:15ae4ca925ecf9cfdfb7a709250846fbb08072260fca08ade78056c502b86bed", - "sha256:43621514301a7e9f5d06dd8013a1b450f30c2e9372b8e30aaeb4562abf2ce034" - ], - "markers": "python_version >= '3.7'", - "version": "==1.22.0" - }, - "opentelemetry-distro": { - "hashes": [ - "sha256:a33f4a5f40f51dbb9f1d21b426d128312f9e2a12fe04da31a9017da6c20c958e", - "sha256:ab4876203d06d40c77cffe1e79c859cff6b471725a846076946cc94df1a863a9" - ], - "markers": "python_version >= '3.7'", - "version": "==0.43b0" - }, - "opentelemetry-exporter-otlp": { - "hashes": [ - "sha256:309a7d4dc67602801f15818e110ce452e78989886aaab5d37e7cf7f55f1d3d27", - "sha256:cb03a1cbf300e12b47690858be13dd26fe2f60b2610204959f3497cd6645e3a1" - ], - "markers": "python_version >= '3.7'", - "version": "==1.22.0" - }, - "opentelemetry-exporter-otlp-proto-common": { - "hashes": [ - "sha256:3f2538bec5312587f8676c332b3747f54c89fe6364803a807e217af4603201fa", - "sha256:71ae2f81bc6d6fe408d06388826edc8933759b2ca3a97d24054507dc7cfce52d" - ], - "markers": "python_version >= '3.7'", - "version": "==1.22.0" - }, - "opentelemetry-exporter-otlp-proto-grpc": { - "hashes": [ - "sha256:1e0e5aa4bbabc74942f06f268deffd94851d12a8dc30b02527472ef1729fe5b1", - "sha256:b5bcadc129272004316a455e9081216d3380c1fc2231a928ea6a70aa90e173fb" - ], - "markers": "python_version >= '3.7'", - "version": "==1.22.0" - }, - "opentelemetry-exporter-otlp-proto-http": { - "hashes": [ - "sha256:79ed108981ec68d5f7985355bca32003c2f3a5be1534a96d62d5861b758a82f4", - "sha256:e002e842190af45b91dc55a97789d0b98e4308c88d886b16049ee90e17a4d396" - ], - "markers": "python_version >= '3.7'", - "version": "==1.22.0" - }, - "opentelemetry-instrumentation": { - "hashes": [ - "sha256:0ff1334d7e359e27640e9d420024efeb73eacae464309c2e14ede7ba6c93967e", - "sha256:c3755da6c4be8033be0216d0501e11f4832690f4e2eca5a3576fbf113498f0f6" - ], - "markers": "python_version >= '3.7'", - "version": "==0.43b0" - }, - "opentelemetry-instrumentation-wsgi": { - "hashes": [ - "sha256:0b7511469daa29a6e75b9cc54b4d01a9bb46aa1f964471dc3ee3f06ff39f94b2", - "sha256:3a1cf045f7ccf04987a89cdd49eda93e9195de4c8b73be228a9e565ec3ab453c" - ], - "markers": "python_version >= '3.7'", - "version": "==0.43b0" - }, - "opentelemetry-propagator-aws-xray": { - "hashes": [ - "sha256:49267a1d72b3f04880ac75e24f9ef38fe323e2f3156c4531e0e00c71c0829c0f", - "sha256:6e8be667bbcf17c3d81d70b2a7cdec0b11257ff64d3829ffe75b810ba1b49f86" - ], - "markers": "python_version >= '3.6'", - "version": "==1.0.1" - }, - "opentelemetry-proto": { - "hashes": [ - "sha256:9ec29169286029f17ca34ec1f3455802ffb90131642d2f545ece9a63e8f69003", - "sha256:ce7188d22c75b6d0fe53e7fb58501613d0feade5139538e79dedd9420610fa0c" - ], - "markers": "python_version >= '3.7'", - "version": "==1.22.0" - }, - "opentelemetry-sdk": { - "hashes": [ - "sha256:45267ac1f38a431fc2eb5d6e0c0d83afc0b78de57ac345488aa58c28c17991d0", - "sha256:a730555713d7c8931657612a88a141e3a4fe6eb5523d9e2d5a8b1e673d76efa6" - ], - "markers": "python_version >= '3.7'", - "version": "==1.22.0" - }, - "opentelemetry-sdk-extension-aws": { - "hashes": [ - "sha256:dd7cf6fc0e7c8070dbe179348f2f194ca4555601b60efb7264d82fc8df53f4ba", - "sha256:f964b0598793ded268d3329c33829fad33f63a8d9299fe51bf3a743e81fd7c67" - ], - "markers": "python_version >= '3.6'", - "version": "==2.0.1" - }, - "opentelemetry-semantic-conventions": { - "hashes": [ - "sha256:291284d7c1bf15fdaddf309b3bd6d3b7ce12a253cec6d27144439819a15d8445", - "sha256:b9576fb890df479626fa624e88dde42d3d60b8b6c8ae1152ad157a8b97358635" - ], - "markers": "python_version >= '3.7'", - "version": "==0.43b0" - }, - "opentelemetry-util-http": { - "hashes": [ - "sha256:3ff6ab361dbe99fc81200d625603c0fb890c055c6e416a3e6d661ddf47a6c7f7", - "sha256:f25a820784b030f6cb86b3d76e5676c769b75ed3f55a210bcdae0a5e175ebadb" - ], - "markers": "python_version >= '3.7'", - "version": "==0.43b0" - }, "packaging": { "hashes": [ "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002", @@ -1505,23 +1316,6 @@ "markers": "python_version >= '3.8'", "version": "==4.25.3" }, - "protobuf": { - "hashes": [ - "sha256:19b270aeaa0099f16d3ca02628546b8baefe2955bbe23224aaf856134eccf1e4", - "sha256:209ba4cc916bab46f64e56b85b090607a676f66b473e6b762e6f1d9d591eb2e8", - "sha256:25b5d0b42fd000320bd7830b349e3b696435f3b329810427a6bcce6a5492cc5c", - "sha256:7c8daa26095f82482307bc717364e7c13f4f1c99659be82890dcfc215194554d", - "sha256:c053062984e61144385022e53678fbded7aea14ebb3e0305ae3592fb219ccfa4", - "sha256:d4198877797a83cbfe9bffa3803602bbe1625dc30d8a097365dbc762e5790faa", - "sha256:e3c97a1555fd6388f857770ff8b9703083de6bf1f9274a002a332d65fbb56c8c", - "sha256:e7cb0ae90dd83727f0c0718634ed56837bfeeee29a5f82a7514c03ee1364c019", - "sha256:f0700d54bcf45424477e46a9f0944155b46fb0639d69728739c0e47bab83f2b9", - "sha256:f1279ab38ecbfae7e456a108c5c0681e4956d5b1090027c1de0f934dfdb4b35c", - "sha256:f4f118245c4a087776e0a8408be33cf09f6c547442c00395fbfb116fac2f8ac2" - ], - "markers": "python_version >= '3.8'", - "version": "==4.25.3" - }, "psycopg": { "hashes": [ "sha256:92d7b78ad82426cdcf1a0440678209faa890c6e1721361c2f8901f0dccd62961", @@ -1592,7 +1386,7 @@ "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==2.9.0.post0" }, "pytz": { @@ -1713,19 +1507,16 @@ "hashes": [ "sha256:01a1e793faa5bd89abc851fa15d0a0db26f160890c7102cd8dce643e886b47f5", "sha256:d9b8b771455a97c8a9f3ab3448ebe0b29b5e105f1228bba41028be116985a267" - "sha256:01a1e793faa5bd89abc851fa15d0a0db26f160890c7102cd8dce643e886b47f5", - "sha256:d9b8b771455a97c8a9f3ab3448ebe0b29b5e105f1228bba41028be116985a267" ], "markers": "python_version >= '3.8'", "version": "==70.1.0" - "version": "==70.1.0" }, "six": { "hashes": [ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==1.16.0" }, "sqlparse": { @@ -1798,7 +1589,7 @@ "sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3", "sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "markers": "python_version >= '3.6'", "version": "==1.26.19" }, "vine": { @@ -1925,82 +1716,6 @@ "markers": "python_version >= '3.6'", "version": "==1.16.0" }, - "wrapt": { - "hashes": [ - "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc", - "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81", - "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09", - "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e", - "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca", - "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0", - "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb", - "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487", - "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40", - "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c", - "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060", - "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202", - "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41", - "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9", - "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b", - "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664", - "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d", - "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362", - "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00", - "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc", - "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1", - "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267", - "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956", - "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966", - "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1", - "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228", - "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72", - "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d", - "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292", - "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0", - "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0", - "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36", - "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c", - "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5", - "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f", - "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73", - "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b", - "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2", - "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593", - "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39", - "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389", - "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf", - "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf", - "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89", - "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c", - "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c", - "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f", - "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440", - "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465", - "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136", - "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b", - "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8", - "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3", - "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8", - "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6", - "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e", - "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f", - "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c", - "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e", - "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8", - "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2", - "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020", - "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35", - "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d", - "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3", - "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537", - "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809", - "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d", - "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a", - "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4" - ], - "markers": "python_version >= '3.6'", - "version": "==1.16.0" - }, "xmlschema": { "hashes": [ "sha256:1c4515dd16b5b556dac27cae70bdb3c863e1448e9d58896326ff468ff55e4246", @@ -2709,7 +2424,7 @@ "sha256:06d39a8b70fde873eb2a131141a0e79bb34a432941fb3d66fad247abafc9766c", "sha256:79b1f2497060d0928bc46016793f1fca1057c4aacdf15ef876aa48d75a73a355" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1'", "version": "==0.6.2" }, "parso": { @@ -2927,7 +2642,7 @@ "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==2.9.0.post0" }, "pyyaml": { @@ -3048,19 +2763,16 @@ "hashes": [ "sha256:01a1e793faa5bd89abc851fa15d0a0db26f160890c7102cd8dce643e886b47f5", "sha256:d9b8b771455a97c8a9f3ab3448ebe0b29b5e105f1228bba41028be116985a267" - "sha256:01a1e793faa5bd89abc851fa15d0a0db26f160890c7102cd8dce643e886b47f5", - "sha256:d9b8b771455a97c8a9f3ab3448ebe0b29b5e105f1228bba41028be116985a267" ], "markers": "python_version >= '3.8'", "version": "==70.1.0" - "version": "==70.1.0" }, "six": { "hashes": [ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==1.16.0" }, "smmap": { @@ -3091,7 +2803,7 @@ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", "version": "==0.10.2" }, "tomli": { @@ -3123,7 +2835,7 @@ "sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3", "sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "markers": "python_version >= '3.6'", "version": "==1.26.19" }, "watchdog": { @@ -3254,81 +2966,9 @@ "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d", "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a", "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4" - "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc", - "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81", - "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09", - "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e", - "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca", - "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0", - "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb", - "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487", - "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40", - "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c", - "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060", - "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202", - "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41", - "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9", - "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b", - "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664", - "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d", - "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362", - "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00", - "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc", - "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1", - "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267", - "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956", - "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966", - "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1", - "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228", - "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72", - "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d", - "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292", - "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0", - "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0", - "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36", - "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c", - "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5", - "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f", - "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73", - "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b", - "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2", - "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593", - "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39", - "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389", - "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf", - "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf", - "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89", - "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c", - "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c", - "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f", - "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440", - "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465", - "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136", - "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b", - "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8", - "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3", - "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8", - "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6", - "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e", - "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f", - "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c", - "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e", - "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8", - "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2", - "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020", - "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35", - "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d", - "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3", - "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537", - "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809", - "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d", - "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a", - "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4" ], "markers": "python_version >= '3.6'", "version": "==1.16.0" - "markers": "python_version >= '3.6'", - "version": "==1.16.0" }, "xmltodict": { "hashes": [ From e5737b9ac52239035c1102e40172a4b177e3c5f2 Mon Sep 17 00:00:00 2001 From: "mark.j0hnst0n" Date: Fri, 21 Jun 2024 13:08:26 +0100 Subject: [PATCH 24/73] remove unused --- api/conf/settings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/api/conf/settings.py b/api/conf/settings.py index da18b4f10a..40f28663a4 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -9,7 +9,6 @@ from sentry_sdk.integrations.django import DjangoIntegration from dbt_copilot_python.network import setup_allowed_hosts -from dbt_copilot_python.utility import is_copilot from django_log_formatter_ecs import ECSFormatter from django_log_formatter_asim import ASIMFormatter From 18c8bb4cdc20146ae6f92f78f830b3181fb6a93c Mon Sep 17 00:00:00 2001 From: "mark.j0hnst0n" Date: Mon, 24 Jun 2024 09:21:54 +0100 Subject: [PATCH 25/73] assign env once --- api/conf/settings.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/api/conf/settings.py b/api/conf/settings.py index 40f28663a4..a3d86ac6fc 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -379,9 +379,9 @@ def _build_redis_url(base_url, db_number, **query_args): }, } -ENVIRONMENT = env.str("ENV", "") +ENV = env.str("ENV", "") -if ENVIRONMENT == "local": +if ENV == "local": LOGGING.update({"formatters": {"simple": {"format": "{asctime} {levelname} {message}", "style": "{"}}}) LOGGING["handlers"].update({"stdout": {"class": "logging.StreamHandler", "formatter": "simple"}}) LOGGING.update({"root": {"handlers": ["stdout"], "level": env("LOG_LEVEL").upper()}}) @@ -434,8 +434,6 @@ def _build_redis_url(base_url, db_number, **query_args): GOV_NOTIFY_KEY = env("GOV_NOTIFY_KEY") -ENV = env("ENV") - # If EXPORTER_BASE_URL is not in env vars, build the base_url using the environment EXPORTER_BASE_URL = env("EXPORTER_BASE_URL") or f"https://exporter.lite.service.{ENV}.uktrade.digital" From facc056a8c254f12d17b0553dd13763e10747f18 Mon Sep 17 00:00:00 2001 From: "mark.j0hnst0n" Date: Mon, 24 Jun 2024 10:55:51 +0100 Subject: [PATCH 26/73] env var changes --- api/conf/settings.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/conf/settings.py b/api/conf/settings.py index a3d86ac6fc..2d9c760838 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -9,6 +9,7 @@ from sentry_sdk.integrations.django import DjangoIntegration from dbt_copilot_python.network import setup_allowed_hosts +from dbt_copilot_python.utility import is_copilot from django_log_formatter_ecs import ECSFormatter from django_log_formatter_asim import ASIMFormatter @@ -379,9 +380,7 @@ def _build_redis_url(base_url, db_number, **query_args): }, } -ENV = env.str("ENV", "") - -if ENV == "local": +if env.str("ENV") == "local": LOGGING.update({"formatters": {"simple": {"format": "{asctime} {levelname} {message}", "style": "{"}}}) LOGGING["handlers"].update({"stdout": {"class": "logging.StreamHandler", "formatter": "simple"}}) LOGGING.update({"root": {"handlers": ["stdout"], "level": env("LOG_LEVEL").upper()}}) @@ -434,6 +433,7 @@ def _build_redis_url(base_url, db_number, **query_args): GOV_NOTIFY_KEY = env("GOV_NOTIFY_KEY") +ENV = env("ENV") # If EXPORTER_BASE_URL is not in env vars, build the base_url using the environment EXPORTER_BASE_URL = env("EXPORTER_BASE_URL") or f"https://exporter.lite.service.{ENV}.uktrade.digital" From bfa6c286a6cc169de0ac855cfe05cced45bf08a4 Mon Sep 17 00:00:00 2001 From: "mark.j0hnst0n" Date: Mon, 24 Jun 2024 10:57:10 +0100 Subject: [PATCH 27/73] remove unused again --- api/conf/settings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/api/conf/settings.py b/api/conf/settings.py index 2d9c760838..e2b09c9912 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -13,7 +13,6 @@ from django_log_formatter_ecs import ECSFormatter from django_log_formatter_asim import ASIMFormatter -from dbt_copilot_python.utility import is_copilot from django.urls import reverse_lazy From bbb63205570eb41dd2d6dffa168ece534889f6c2 Mon Sep 17 00:00:00 2001 From: "mark.j0hnst0n" Date: Mon, 24 Jun 2024 12:29:49 +0100 Subject: [PATCH 28/73] update env again --- api/conf/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/conf/settings.py b/api/conf/settings.py index e2b09c9912..18852bb95a 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -365,6 +365,7 @@ def _build_redis_url(base_url, db_number, **query_args): "django_elasticsearch_dsl_drf", ] +ENV = env("ENV") DENIAL_REASONS_DELETION_LOGGER = "denial_reasons_deletion_logger" @@ -379,7 +380,7 @@ def _build_redis_url(base_url, db_number, **query_args): }, } -if env.str("ENV") == "local": +if ENV == "local": LOGGING.update({"formatters": {"simple": {"format": "{asctime} {levelname} {message}", "style": "{"}}}) LOGGING["handlers"].update({"stdout": {"class": "logging.StreamHandler", "formatter": "simple"}}) LOGGING.update({"root": {"handlers": ["stdout"], "level": env("LOG_LEVEL").upper()}}) @@ -432,7 +433,6 @@ def _build_redis_url(base_url, db_number, **query_args): GOV_NOTIFY_KEY = env("GOV_NOTIFY_KEY") -ENV = env("ENV") # If EXPORTER_BASE_URL is not in env vars, build the base_url using the environment EXPORTER_BASE_URL = env("EXPORTER_BASE_URL") or f"https://exporter.lite.service.{ENV}.uktrade.digital" From d392d3e984950c218c935c4154567a20eda85032 Mon Sep 17 00:00:00 2001 From: Brendan Smith Date: Tue, 18 Jun 2024 14:23:24 +0100 Subject: [PATCH 29/73] Ensure that creating an amendment saves an audit trail entry for status change --- api/applications/models.py | 9 ++++----- api/applications/tests/test_models.py | 4 ++++ api/cases/models.py | 13 +------------ api/cases/views/views.py | 19 +++++++++++++++---- 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/api/applications/models.py b/api/applications/models.py index ac93c768e9..84b557fb03 100644 --- a/api/applications/models.py +++ b/api/applications/models.py @@ -42,7 +42,8 @@ from api.staticdata.statuses.libraries.get_case_status import get_case_status_by_status from api.staticdata.trade_control.enums import TradeControlProductCategory, TradeControlActivity from api.staticdata.units.enums import Units -from api.users.models import ExporterUser, GovUser +from api.users.enums import SystemUser +from api.users.models import ExporterUser, GovUser, BaseUser from lite_content.lite_api.strings import PartyErrors from lite_routing.routing_rules_internal.enums import QueuesEnum @@ -361,11 +362,9 @@ def clone(self, exclusions=None, **overrides): @transaction.atomic def create_amendment(self): amendment_application = self.clone(amendment_of=self) - # TODO: Do we need a log on the audit trail? - # Remove case from all queues and set status to superseded CaseQueue.objects.filter(case=self.case_ptr).delete() - self.status = get_case_status_by_status(CaseStatusEnum.SUPERSEDED_BY_AMENDMENT) - self.save() + system_user = BaseUser.objects.get(id=SystemUser.id) + self.case_ptr.change_status(system_user, get_case_status_by_status(CaseStatusEnum.SUPERSEDED_BY_AMENDMENT)) return amendment_application diff --git a/api/applications/tests/test_models.py b/api/applications/tests/test_models.py index 75d5fd5ea4..0d9e42d95a 100644 --- a/api/applications/tests/test_models.py +++ b/api/applications/tests/test_models.py @@ -5,6 +5,7 @@ from test_helpers.clients import DataTestClient from api.appeals.tests.factories import AppealFactory +from api.audit_trail.models import Audit from api.cases.models import CaseType, Queue from api.flags.models import Flag from api.applications.models import ( @@ -53,6 +54,9 @@ def test_create_amendment(self): original_application.refresh_from_db() assert original_application.status.status == "superseded_by_amendment" assert original_application.queues.all().count() == 0 + audit_entry = Audit.objects.first() + assert audit_entry.payload == {"status": {"new": "Superseded by amendment", "old": "ogd_advice"}} + assert audit_entry.verb == "updated_status" def test_clone(self): original_application = StandardApplicationFactory( diff --git a/api/cases/models.py b/api/cases/models.py index ea771561b6..bae1c7cccd 100644 --- a/api/cases/models.py +++ b/api/cases/models.py @@ -45,6 +45,7 @@ CaseSubStatus, ) from api.teams.models import Team, Department +from api.users.enums import SystemUser from api.users.models import ( BaseUser, ExporterUser, @@ -198,25 +199,13 @@ def change_status(self, user, status: CaseStatus, note: Optional[str] = ""): Sets the status for the case, runs validation on various parameters, creates audit entries and also runs flagging and automation rules """ - from api.cases.helpers import can_set_status from api.audit_trail import service as audit_trail_service - from api.applications.libraries.application_helpers import can_status_be_set_by_gov_user from api.workflow.flagging_rules_automation import apply_flagging_rules_to_case from api.licences.helpers import update_licence_status from lite_routing.routing_rules_internal.routing_engine import run_routing_rules old_status = self.status.status - # Only allow the final decision if the user has the MANAGE_FINAL_ADVICE permission - if status.status == CaseStatusEnum.FINALISED: - assert_user_has_permission(user.govuser, GovPermissions.MANAGE_LICENCE_FINAL_ADVICE) - - if not can_set_status(self, status.status): - raise ValidationError({"status": [strings.Statuses.BAD_STATUS]}) - - if not can_status_be_set_by_gov_user(user.govuser, old_status, status.status, is_mod=False): - raise ValidationError({"status": ["Status cannot be set by user"]}) - self.status = status self.save() diff --git a/api/cases/views/views.py b/api/cases/views/views.py index b2ed12a634..ac30c83299 100644 --- a/api/cases/views/views.py +++ b/api/cases/views/views.py @@ -18,6 +18,7 @@ CountryWithFlagsSerializer, CountersignDecisionAdviceSerializer, ) +from api.applications.libraries.application_helpers import can_status_be_set_by_gov_user from api.audit_trail import service as audit_trail_service from api.audit_trail.enums import AuditType from api.cases import notify @@ -28,7 +29,7 @@ ) from api.cases.generated_documents.models import GeneratedCaseDocument from api.cases.generated_documents.serializers import AdviceDocumentGovSerializer -from api.cases.helpers import create_system_mention +from api.cases.helpers import create_system_mention, can_set_status from api.cases.libraries.advice import group_advice from api.cases.libraries.finalise import ( get_required_decision_document_types, @@ -186,9 +187,19 @@ def patch(self, request, pk): Change case status """ case = get_case(pk) - case.change_status( - request.user, get_case_status_by_status(request.data.get("status")), request.data.get("note") - ) + new_status = get_case_status_by_status(request.data.get("status")) + + # Only allow the final decision if the user has the MANAGE_FINAL_ADVICE permission + if new_status.status == CaseStatusEnum.FINALISED: + assert_user_has_permission(user.govuser, GovPermissions.MANAGE_LICENCE_FINAL_ADVICE) + + if not can_set_status(case, new_status.status): + raise ValidationError({"status": [strings.Statuses.BAD_STATUS]}) + + if not can_status_be_set_by_gov_user(request.user.govuser, case.status.status, new_status.status, is_mod=False): + raise ValidationError({"status": ["Status cannot be set by user"]}) + + case.change_status(request.user, new_status, request.data.get("note")) return JsonResponse(data={}, status=status.HTTP_200_OK) From b18bdb61d7283ba7a2a1735bdb78e458fa124e25 Mon Sep 17 00:00:00 2001 From: Brendan Smith Date: Tue, 18 Jun 2024 15:13:41 +0100 Subject: [PATCH 30/73] Create an audit trail entry when creating an amendment --- api/applications/models.py | 10 +- api/applications/tests/test_models.py | 14 +- api/applications/views/amendments.py | 2 +- api/audit_trail/enums.py | 1 + .../migrations/0023_alter_audit_verb.py | 277 ++++++++++++++++++ api/audit_trail/payload.py | 1 + api/cases/tests/test_get_case.py | 4 +- api/cases/tests/test_models.py | 4 +- api/cases/tests/test_status.py | 2 + 9 files changed, 306 insertions(+), 9 deletions(-) create mode 100644 api/audit_trail/migrations/0023_alter_audit_verb.py diff --git a/api/applications/models.py b/api/applications/models.py index 84b557fb03..dbdd06e3da 100644 --- a/api/applications/models.py +++ b/api/applications/models.py @@ -249,7 +249,7 @@ def set_appealed(self, appeal, exporter_user): self.set_sub_status(CaseSubStatusIdEnum.UNDER_APPEAL__APPEAL_RECEIVED) self.add_to_queue(Queue.objects.get(id=QueuesEnum.LU_APPEALS)) - def create_amendment(self): + def create_amendment(self, user): raise NotImplementedError() @@ -360,9 +360,15 @@ def clone(self, exclusions=None, **overrides): return cloned_application @transaction.atomic - def create_amendment(self): + def create_amendment(self, user): amendment_application = self.clone(amendment_of=self) CaseQueue.objects.filter(case=self.case_ptr).delete() + audit_trail_service.create( + actor=user, + verb=AuditType.EXPORTER_CREATED_AMENDMENT, + target=self.get_case(), + payload={}, + ) system_user = BaseUser.objects.get(id=SystemUser.id) self.case_ptr.change_status(system_user, get_case_status_by_status(CaseStatusEnum.SUPERSEDED_BY_AMENDMENT)) return amendment_application diff --git a/api/applications/tests/test_models.py b/api/applications/tests/test_models.py index 0d9e42d95a..52b1295789 100644 --- a/api/applications/tests/test_models.py +++ b/api/applications/tests/test_models.py @@ -45,7 +45,8 @@ def test_create_amendment(self): original_application.queues.add(Queue.objects.first()) original_application.save() - amendment_application = original_application.create_amendment() + exporter_user = ExporterUser.objects.first() + amendment_application = original_application.create_amendment(exporter_user) # Ensure the amendment application has been saved to the DB - by retrieving it directly amendment_application = StandardApplication.objects.get(id=amendment_application.id) # It's unnecessary to be exhaustive in testing clone functionality as that is done below @@ -54,9 +55,14 @@ def test_create_amendment(self): original_application.refresh_from_db() assert original_application.status.status == "superseded_by_amendment" assert original_application.queues.all().count() == 0 - audit_entry = Audit.objects.first() - assert audit_entry.payload == {"status": {"new": "Superseded by amendment", "old": "ogd_advice"}} - assert audit_entry.verb == "updated_status" + audit_entries = Audit.objects.all() + amendment_audit_entry = audit_entries[1] + assert amendment_audit_entry.payload == {} + assert amendment_audit_entry.verb == "exporter_created_amendment" + assert amendment_audit_entry.actor == exporter_user + status_change_audit_entry = audit_entries[0] + assert status_change_audit_entry.payload == {"status": {"new": "Superseded by amendment", "old": "ogd_advice"}} + assert status_change_audit_entry.verb == "updated_status" def test_clone(self): original_application = StandardApplicationFactory( diff --git a/api/applications/views/amendments.py b/api/applications/views/amendments.py index fc83b44d78..ca0b3f14d1 100644 --- a/api/applications/views/amendments.py +++ b/api/applications/views/amendments.py @@ -35,7 +35,7 @@ def get_organisation(self): return self.application.organisation def perform_create(self, serializer): - self.amendment_application = self.application.create_amendment() + self.amendment_application = self.application.create_amendment(self.request.user) def create(self, request, *args, **kwargs): super().create(request, *args, **kwargs) diff --git a/api/audit_trail/enums.py b/api/audit_trail/enums.py index be5c78fc39..6f280ef19b 100644 --- a/api/audit_trail/enums.py +++ b/api/audit_trail/enums.py @@ -141,6 +141,7 @@ class AuditType(LiteEnum): LU_CREATE_MEETING_NOTE = autostr() CREATE_REFUSAL_CRITERIA = autostr() EXPORTER_APPEALED_REFUSAL = autostr() + EXPORTER_CREATED_AMENDMENT = autostr() def human_readable(self): """ diff --git a/api/audit_trail/migrations/0023_alter_audit_verb.py b/api/audit_trail/migrations/0023_alter_audit_verb.py new file mode 100644 index 0000000000..5a57ac9241 --- /dev/null +++ b/api/audit_trail/migrations/0023_alter_audit_verb.py @@ -0,0 +1,277 @@ +# Generated by Django 4.2.13 on 2024-06-18 13:55 + +import api.audit_trail.enums +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("audit_trail", "0022_alter_audit_verb"), + ] + + operations = [ + migrations.AlterField( + model_name="audit", + name="verb", + field=models.CharField( + choices=[ + (api.audit_trail.enums.AuditType["CREATED"], "created"), + (api.audit_trail.enums.AuditType["OGL_CREATED"], "ogl_created"), + (api.audit_trail.enums.AuditType["OGL_FIELD_EDITED"], "ogl_field_edited"), + (api.audit_trail.enums.AuditType["OGL_MULTI_FIELD_EDITED"], "ogl_multi_field_edited"), + (api.audit_trail.enums.AuditType["ADD_FLAGS"], "add_flags"), + (api.audit_trail.enums.AuditType["REMOVE_FLAGS"], "remove_flags"), + (api.audit_trail.enums.AuditType["GOOD_REVIEWED"], "good_reviewed"), + (api.audit_trail.enums.AuditType["GOOD_ADD_FLAGS"], "good_add_flags"), + (api.audit_trail.enums.AuditType["GOOD_REMOVE_FLAGS"], "good_remove_flags"), + (api.audit_trail.enums.AuditType["GOOD_ADD_REMOVE_FLAGS"], "good_add_remove_flags"), + (api.audit_trail.enums.AuditType["DESTINATION_ADD_FLAGS"], "destination_add_flags"), + (api.audit_trail.enums.AuditType["DESTINATION_REMOVE_FLAGS"], "destination_remove_flags"), + (api.audit_trail.enums.AuditType["ADD_GOOD_TO_APPLICATION"], "add_good_to_application"), + (api.audit_trail.enums.AuditType["REMOVE_GOOD_FROM_APPLICATION"], "remove_good_from_application"), + (api.audit_trail.enums.AuditType["ADD_GOOD_TYPE_TO_APPLICATION"], "add_good_type_to_application"), + ( + api.audit_trail.enums.AuditType["REMOVE_GOOD_TYPE_FROM_APPLICATION"], + "remove_good_type_from_application", + ), + ( + api.audit_trail.enums.AuditType["UPDATE_APPLICATION_END_USE_DETAIL"], + "update_application_end_use_detail", + ), + ( + api.audit_trail.enums.AuditType["UPDATE_APPLICATION_TEMPORARY_EXPORT"], + "update_application_temporary_export", + ), + ( + api.audit_trail.enums.AuditType["REMOVED_SITES_FROM_APPLICATION"], + "removed_sites_from_application", + ), + (api.audit_trail.enums.AuditType["ADD_SITES_TO_APPLICATION"], "add_sites_to_application"), + ( + api.audit_trail.enums.AuditType["REMOVED_EXTERNAL_LOCATIONS_FROM_APPLICATION"], + "removed_external_locations_from_application", + ), + ( + api.audit_trail.enums.AuditType["ADD_EXTERNAL_LOCATIONS_TO_APPLICATION"], + "add_external_locations_to_application", + ), + ( + api.audit_trail.enums.AuditType["REMOVED_COUNTRIES_FROM_APPLICATION"], + "removed_countries_from_application", + ), + (api.audit_trail.enums.AuditType["ADD_COUNTRIES_TO_APPLICATION"], "add_countries_to_application"), + ( + api.audit_trail.enums.AuditType["ADD_ADDITIONAL_CONTACT_TO_CASE"], + "add_additional_contact_to_case", + ), + (api.audit_trail.enums.AuditType["MOVE_CASE"], "move_case"), + (api.audit_trail.enums.AuditType["ASSIGN_CASE"], "assign_case"), + (api.audit_trail.enums.AuditType["ASSIGN_USER_TO_CASE"], "assign_user_to_case"), + (api.audit_trail.enums.AuditType["REMOVE_USER_FROM_CASE"], "remove_user_from_case"), + (api.audit_trail.enums.AuditType["REMOVE_CASE"], "remove_case"), + (api.audit_trail.enums.AuditType["REMOVE_CASE_FROM_ALL_QUEUES"], "remove_case_from_all_queues"), + ( + api.audit_trail.enums.AuditType["REMOVE_CASE_FROM_ALL_USER_ASSIGNMENTS"], + "remove_case_from_all_user_assignments", + ), + (api.audit_trail.enums.AuditType["CLC_RESPONSE"], "clc_response"), + (api.audit_trail.enums.AuditType["PV_GRADING_RESPONSE"], "pv_grading_response"), + (api.audit_trail.enums.AuditType["CREATED_CASE_NOTE"], "created_case_note"), + ( + api.audit_trail.enums.AuditType["CREATED_CASE_NOTE_WITH_MENTIONS"], + "created_case_note_with_mentions", + ), + (api.audit_trail.enums.AuditType["ECJU_QUERY"], "ecju_query"), + (api.audit_trail.enums.AuditType["ECJU_QUERY_RESPONSE"], "ecju_query_response"), + (api.audit_trail.enums.AuditType["ECJU_QUERY_MANUALLY_CLOSED"], "ecju_query_manually_closed"), + (api.audit_trail.enums.AuditType["UPDATED_STATUS"], "updated_status"), + (api.audit_trail.enums.AuditType["UPDATED_SUB_STATUS"], "updated_sub_status"), + (api.audit_trail.enums.AuditType["UPDATED_APPLICATION_NAME"], "updated_application_name"), + ( + api.audit_trail.enums.AuditType["UPDATE_APPLICATION_LETTER_REFERENCE"], + "update_application_letter_reference", + ), + ( + api.audit_trail.enums.AuditType["UPDATE_APPLICATION_F680_CLEARANCE_TYPES"], + "update_application_f680_clearance_types", + ), + ( + api.audit_trail.enums.AuditType["ADDED_APPLICATION_LETTER_REFERENCE"], + "added_application_letter_reference", + ), + ( + api.audit_trail.enums.AuditType["REMOVED_APPLICATION_LETTER_REFERENCE"], + "removed_application_letter_reference", + ), + (api.audit_trail.enums.AuditType["ASSIGNED_COUNTRIES_TO_GOOD"], "assigned_countries_to_good"), + (api.audit_trail.enums.AuditType["REMOVED_COUNTRIES_FROM_GOOD"], "removed_countries_from_good"), + (api.audit_trail.enums.AuditType["CREATED_FINAL_ADVICE"], "created_final_advice"), + (api.audit_trail.enums.AuditType["CLEARED_FINAL_ADVICE"], "cleared_final_advice"), + (api.audit_trail.enums.AuditType["CREATED_TEAM_ADVICE"], "created_team_advice"), + (api.audit_trail.enums.AuditType["CLEARED_TEAM_ADVICE"], "cleared_team_advice"), + (api.audit_trail.enums.AuditType["REVIEW_COMBINE_ADVICE"], "review_combine_advice"), + (api.audit_trail.enums.AuditType["CREATED_USER_ADVICE"], "created_user_advice"), + (api.audit_trail.enums.AuditType["CLEARED_USER_ADVICE"], "cleared_user_advice"), + (api.audit_trail.enums.AuditType["ADD_PARTY"], "add_party"), + (api.audit_trail.enums.AuditType["REMOVE_PARTY"], "remove_party"), + (api.audit_trail.enums.AuditType["UPLOAD_PARTY_DOCUMENT"], "upload_party_document"), + (api.audit_trail.enums.AuditType["DELETE_PARTY_DOCUMENT"], "delete_party_document"), + (api.audit_trail.enums.AuditType["UPLOAD_APPLICATION_DOCUMENT"], "upload_application_document"), + (api.audit_trail.enums.AuditType["DELETE_APPLICATION_DOCUMENT"], "delete_application_document"), + (api.audit_trail.enums.AuditType["UPLOAD_CASE_DOCUMENT"], "upload_case_document"), + (api.audit_trail.enums.AuditType["GENERATE_CASE_DOCUMENT"], "generate_case_document"), + (api.audit_trail.enums.AuditType["ADD_CASE_OFFICER_TO_CASE"], "add_case_officer_to_case"), + (api.audit_trail.enums.AuditType["REMOVE_CASE_OFFICER_FROM_CASE"], "remove_case_officer_from_case"), + (api.audit_trail.enums.AuditType["GRANTED_APPLICATION"], "granted_application"), + (api.audit_trail.enums.AuditType["REINSTATED_APPLICATION"], "reinstated_application"), + (api.audit_trail.enums.AuditType["FINALISED_APPLICATION"], "finalised_application"), + (api.audit_trail.enums.AuditType["UNASSIGNED_QUEUES"], "unassigned_queues"), + (api.audit_trail.enums.AuditType["UNASSIGNED"], "unassigned"), + (api.audit_trail.enums.AuditType["CREATED_DOCUMENT_TEMPLATE"], "created_document_template"), + (api.audit_trail.enums.AuditType["UPDATED_LETTER_TEMPLATE_NAME"], "updated_letter_template_name"), + ( + api.audit_trail.enums.AuditType["ADDED_LETTER_TEMPLATE_CASE_TYPES"], + "added_letter_template_case_types", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_LETTER_TEMPLATE_CASE_TYPES"], + "updated_letter_template_case_types", + ), + ( + api.audit_trail.enums.AuditType["REMOVED_LETTER_TEMPLATE_CASE_TYPES"], + "removed_letter_template_case_types", + ), + ( + api.audit_trail.enums.AuditType["ADDED_LETTER_TEMPLATE_DECISIONS"], + "added_letter_template_decisions", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_LETTER_TEMPLATE_DECISIONS"], + "updated_letter_template_decisions", + ), + ( + api.audit_trail.enums.AuditType["REMOVED_LETTER_TEMPLATE_DECISIONS"], + "removed_letter_template_decisions", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_LETTER_TEMPLATE_PARAGRAPHS"], + "updated_letter_template_paragraphs", + ), + ( + api.audit_trail.enums.AuditType["REMOVED_LETTER_TEMPLATE_PARAGRAPHS"], + "removed_letter_template_paragraphs", + ), + ( + api.audit_trail.enums.AuditType["ADDED_LETTER_TEMPLATE_PARAGRAPHS"], + "added_letter_template_paragraphs", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_LETTER_TEMPLATE_LAYOUT"], + "updated_letter_template_layout", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_LETTER_TEMPLATE_PARAGRAPHS_ORDERING"], + "updated_letter_template_paragraphs_ordering", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_LETTER_TEMPLATE_INCLUDE_DIGITAL_SIGNATURE"], + "updated_letter_template_include_digital_signature", + ), + (api.audit_trail.enums.AuditType["CREATED_PICKLIST"], "created_picklist"), + (api.audit_trail.enums.AuditType["UPDATED_PICKLIST_TEXT"], "updated_picklist_text"), + (api.audit_trail.enums.AuditType["UPDATED_PICKLIST_NAME"], "updated_picklist_name"), + (api.audit_trail.enums.AuditType["DEACTIVATE_PICKLIST"], "deactivate_picklist"), + (api.audit_trail.enums.AuditType["REACTIVATE_PICKLIST"], "reactivate_picklist"), + ( + api.audit_trail.enums.AuditType["UPDATED_EXHIBITION_DETAILS_TITLE"], + "updated_exhibition_details_title", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_EXHIBITION_DETAILS_START_DATE"], + "updated_exhibition_details_start_date", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_EXHIBITION_DETAILS_REQUIRED_BY_DATE"], + "updated_exhibition_details_required_by_date", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_EXHIBITION_DETAILS_REASON_FOR_CLEARANCE"], + "updated_exhibition_details_reason_for_clearance", + ), + (api.audit_trail.enums.AuditType["UPDATED_ROUTE_OF_GOODS"], "updated_route_of_goods"), + (api.audit_trail.enums.AuditType["UPDATED_ORGANISATION"], "updated_organisation"), + (api.audit_trail.enums.AuditType["CREATED_ORGANISATION"], "created_organisation"), + (api.audit_trail.enums.AuditType["REGISTER_ORGANISATION"], "register_organisation"), + (api.audit_trail.enums.AuditType["REJECTED_ORGANISATION"], "rejected_organisation"), + (api.audit_trail.enums.AuditType["APPROVED_ORGANISATION"], "approved_organisation"), + (api.audit_trail.enums.AuditType["REMOVED_FLAG_ON_ORGANISATION"], "removed_flag_on_organisation"), + (api.audit_trail.enums.AuditType["ADDED_FLAG_ON_ORGANISATION"], "added_flag_on_organisation"), + (api.audit_trail.enums.AuditType["RERUN_ROUTING_RULES"], "rerun_routing_rules"), + (api.audit_trail.enums.AuditType["ENFORCEMENT_CHECK"], "enforcement_check"), + (api.audit_trail.enums.AuditType["UPDATED_SITE"], "updated_site"), + (api.audit_trail.enums.AuditType["CREATED_SITE"], "created_site"), + (api.audit_trail.enums.AuditType["UPDATED_SITE_NAME"], "updated_site_name"), + (api.audit_trail.enums.AuditType["COMPLIANCE_SITE_CASE_CREATE"], "compliance_site_case_create"), + ( + api.audit_trail.enums.AuditType["COMPLIANCE_SITE_CASE_NEW_LICENCE"], + "compliance_site_case_new_licence", + ), + (api.audit_trail.enums.AuditType["ADDED_NEXT_REVIEW_DATE"], "added_next_review_date"), + (api.audit_trail.enums.AuditType["EDITED_NEXT_REVIEW_DATE"], "edited_next_review_date"), + (api.audit_trail.enums.AuditType["REMOVED_NEXT_REVIEW_DATE"], "removed_next_review_date"), + (api.audit_trail.enums.AuditType["COMPLIANCE_VISIT_CASE_CREATED"], "compliance_visit_case_created"), + (api.audit_trail.enums.AuditType["COMPLIANCE_VISIT_CASE_UPDATED"], "compliance_visit_case_updated"), + ( + api.audit_trail.enums.AuditType["COMPLIANCE_PEOPLE_PRESENT_CREATED"], + "compliance_people_present_created", + ), + ( + api.audit_trail.enums.AuditType["COMPLIANCE_PEOPLE_PRESENT_UPDATED"], + "compliance_people_present_updated", + ), + ( + api.audit_trail.enums.AuditType["COMPLIANCE_PEOPLE_PRESENT_DELETED"], + "compliance_people_present_deleted", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_GOOD_ON_DESTINATION_MATRIX"], + "updated_good_on_destination_matrix", + ), + (api.audit_trail.enums.AuditType["LICENCE_UPDATED_GOOD_USAGE"], "licence_updated_good_usage"), + (api.audit_trail.enums.AuditType["OGEL_REISSUED"], "ogel_reissued"), + (api.audit_trail.enums.AuditType["LICENCE_UPDATED_STATUS"], "licence_updated_status"), + ( + api.audit_trail.enums.AuditType["DOCUMENT_ON_ORGANISATION_CREATE"], + "document_on_organisation_create", + ), + ( + api.audit_trail.enums.AuditType["DOCUMENT_ON_ORGANISATION_DELETE"], + "document_on_organisation_delete", + ), + ( + api.audit_trail.enums.AuditType["DOCUMENT_ON_ORGANISATION_UPDATE"], + "document_on_organisation_update", + ), + (api.audit_trail.enums.AuditType["REPORT_SUMMARY_UPDATED"], "report_summary_updated"), + (api.audit_trail.enums.AuditType["COUNTERSIGN_ADVICE"], "countersign_advice"), + (api.audit_trail.enums.AuditType["UPDATED_SERIAL_NUMBERS"], "updated_serial_numbers"), + (api.audit_trail.enums.AuditType["PRODUCT_REVIEWED"], "product_reviewed"), + (api.audit_trail.enums.AuditType["LICENCE_UPDATED_PRODUCT_USAGE"], "licence_updated_product_usage"), + (api.audit_trail.enums.AuditType["CREATED_FINAL_RECOMMENDATION"], "created_final_recommendation"), + (api.audit_trail.enums.AuditType["GENERATE_DECISION_LETTER"], "generate_decision_letter"), + (api.audit_trail.enums.AuditType["DECISION_LETTER_SENT"], "decision_letter_sent"), + (api.audit_trail.enums.AuditType["LU_ADVICE"], "lu_advice"), + (api.audit_trail.enums.AuditType["LU_EDIT_ADVICE"], "lu_edit_advice"), + (api.audit_trail.enums.AuditType["LU_COUNTERSIGN"], "lu_countersign"), + (api.audit_trail.enums.AuditType["LU_EDIT_MEETING_NOTE"], "lu_edit_meeting_note"), + (api.audit_trail.enums.AuditType["LU_CREATE_MEETING_NOTE"], "lu_create_meeting_note"), + (api.audit_trail.enums.AuditType["CREATE_REFUSAL_CRITERIA"], "create_refusal_criteria"), + (api.audit_trail.enums.AuditType["EXPORTER_APPEALED_REFUSAL"], "exporter_appealed_refusal"), + (api.audit_trail.enums.AuditType["EXPORTER_CREATED_AMENDMENT"], "exporter_created_amendment"), + ], + db_index=True, + max_length=255, + ), + ), + ] diff --git a/api/audit_trail/payload.py b/api/audit_trail/payload.py index 5184caff6c..fcad7773da 100644 --- a/api/audit_trail/payload.py +++ b/api/audit_trail/payload.py @@ -159,4 +159,5 @@ def format_payload(audit_type, payload): AuditType.LU_CREATE_MEETING_NOTE: formatters.create_lu_meeting_note, AuditType.CREATE_REFUSAL_CRITERIA: formatters.create_refusal_criteria, AuditType.EXPORTER_APPEALED_REFUSAL: " appealed refusal", + AuditType.EXPORTER_CREATED_AMENDMENT: " opened the application for a 'major edit'", } diff --git a/api/cases/tests/test_get_case.py b/api/cases/tests/test_get_case.py index 7adb34535f..b704de742e 100644 --- a/api/cases/tests/test_get_case.py +++ b/api/cases/tests/test_get_case.py @@ -21,6 +21,7 @@ from api.staticdata.trade_control.enums import TradeControlActivity, TradeControlProductCategory from test_helpers.clients import DataTestClient from api.staticdata.statuses.models import CaseStatus, CaseSubStatus +from api.users.models import ExporterUser from lite_routing.routing_rules_internal.enums import FlagsEnum @@ -44,7 +45,8 @@ def test_case_endpoint_responds_ok(self): def test_case_endpoint_responds_ok_for_amendment(self): superseded_case = self.submit_application(self.standard_application) - amendment = superseded_case.create_amendment() + exporter_user = ExporterUser.objects.first() + amendment = superseded_case.create_amendment(exporter_user) amendment = self.submit_application(amendment) url = reverse("cases:case", kwargs={"pk": superseded_case.id}) diff --git a/api/cases/tests/test_models.py b/api/cases/tests/test_models.py index dc8fa2d5d0..f1e9b92cff 100644 --- a/api/cases/tests/test_models.py +++ b/api/cases/tests/test_models.py @@ -8,6 +8,7 @@ from api.cases.tests.factories import CaseFactory from api.staticdata.statuses.enums import CaseStatusEnum, CaseSubStatusIdEnum from api.staticdata.statuses.models import CaseStatus, CaseSubStatus +from api.users.models import ExporterUser from test_helpers.clients import DataTestClient @@ -21,7 +22,8 @@ def test_set_sub_status_invalid(self): self.assertRaises(BadSubStatus, self.case.set_sub_status, CaseSubStatusIdEnum.FINALISED__APPROVED) def test_superseded_by_amendment_exists(self): - amendment = self.case.create_amendment() + exporter_user = ExporterUser.objects.first() + amendment = self.case.create_amendment(exporter_user) assert self.case.superseded_by == amendment.case_ptr def test_superseded_by_no_amendment_exists(self): diff --git a/api/cases/tests/test_status.py b/api/cases/tests/test_status.py index eab5c88af4..7f3a25b828 100644 --- a/api/cases/tests/test_status.py +++ b/api/cases/tests/test_status.py @@ -62,6 +62,8 @@ def test_certain_case_statuses_changes_licence_status(self, case_status, licence self.assertEqual(self.case.status.status, case_status) self.assertEqual(licence.status, licence_status) + # TODO: More tests covering different paths for status change view + class EndUserAdvisoryUpdate(DataTestClient): def setUp(self): From a30136f16798bb445c2fb48abf2d4297c7db3174 Mon Sep 17 00:00:00 2001 From: Brendan Smith Date: Tue, 18 Jun 2024 16:13:57 +0100 Subject: [PATCH 31/73] Change application submission audit entry for amendment case --- .../libraries/application_helpers.py | 26 +++++++++++- api/applications/models.py | 7 ++++ api/applications/tests/test_models.py | 42 +++++++++++++++++++ api/applications/views/applications.py | 2 +- api/audit_trail/formatters.py | 2 + 5 files changed, 76 insertions(+), 3 deletions(-) diff --git a/api/applications/libraries/application_helpers.py b/api/applications/libraries/application_helpers.py index 79251f2c1b..ad9e765107 100644 --- a/api/applications/libraries/application_helpers.py +++ b/api/applications/libraries/application_helpers.py @@ -68,9 +68,31 @@ def can_status_be_set_by_gov_user(user: GovUser, original_status: str, new_statu return True -def create_submitted_audit(request: Request, application, old_status: str) -> None: +def create_submitted_audit(user, application, old_status: str, additional_payload=None) -> None: + if not _additional_payload: + additional_payload = {} + + payload = { + "status": { + "new": CaseStatusEnum.RESUBMITTED if old_status != CaseStatusEnum.DRAFT else CaseStatusEnum.SUBMITTED, + "old": old_status, + }, + **additional_payload, + } + + audit_trail_service.create( + actor=user, + verb=AuditType.UPDATED_STATUS, + target=application.get_case(), + payload=payload, + ignore_case_status=True, + send_notification=False, + ) + + +def create_amendment_submitted_audit(user, application, old_status: str) -> None: audit_trail_service.create( - actor=request.user, + actor=user, verb=AuditType.UPDATED_STATUS, target=application.get_case(), payload={ diff --git a/api/applications/models.py b/api/applications/models.py index dbdd06e3da..cd42a10b82 100644 --- a/api/applications/models.py +++ b/api/applications/models.py @@ -16,6 +16,7 @@ ) from api.appeals.models import Appeal from api.applications.managers import BaseApplicationManager +from api.applications.libraries.application_helpers import create_submitted_audit from api.audit_trail.models import Audit, AuditType from api.audit_trail import service as audit_trail_service from api.cases.enums import CaseTypeEnum @@ -200,6 +201,12 @@ class BaseApplication(ApplicationPartyMixin, Case): class Meta: ordering = ["created_at"] + def on_submit(self): + additional_payload = {} + if self.amendment_of: + additional_payload["amendment_of"] = {"reference_code": self.amendment_of.reference_code} + create_submitted_audit(self.submitted_by, application, old_status, additional_payload) + def add_to_queue(self, queue): case = self.get_case() diff --git a/api/applications/tests/test_models.py b/api/applications/tests/test_models.py index 52b1295789..cd6f90cae5 100644 --- a/api/applications/tests/test_models.py +++ b/api/applications/tests/test_models.py @@ -34,6 +34,48 @@ from api.staticdata.regimes.models import RegimeEntry from api.staticdata.report_summaries.models import ReportSummary, ReportSummaryPrefix, ReportSummarySubject from api.staticdata.statuses.models import CaseStatus, CaseSubStatus +from api.users.models import ExporterUser + + +class TestBaseApplication(DataTestClient): + + def test_on_submit_new_application(self): + draft_status = CaseStatus.objects.get(status="draft") + submitted_by = ExporterUser.objects.first() + # Use StandardApplication as BaseApplication is an abstract model + application = StandardApplicationFactory( + status=draft_status, + submitted_by=submitted_by, + ) + application.on_submit(draft_status.status) + submitted_audit_entry = Audit.objects.first() + assert submitted_audit_entry.verb == "updated_status" + assert submitted_audit_entry.payload == {"status": {"new": "submitted", "old": "draft"}} + assert submitted_audit_entry.actor == submitted_by + + def test_on_submit_amendment_application(self): + draft_status = CaseStatus.objects.get(status="draft") + submitted_by = ExporterUser.objects.first() + # Use StandardApplication as BaseApplication is an abstract model + original_application = StandardApplicationFactory() + amendment_application = StandardApplicationFactory( + status=draft_status, + submitted_by=submitted_by, + amendment_of=original_application.case_ptr, + ) + amendment_application.on_submit(draft_status.status) + audit_entries = Audit.objects.all() + submitted_audit_entry = audit_entries[0] + assert submitted_audit_entry.verb == "updated_status" + assert submitted_audit_entry.payload == { + "status": {"new": "submitted", "old": "draft"}, + "amendment_of": {"reference_code": original_application.reference_code}, + } + assert submitted_audit_entry.actor == submitted_by + amendment_audit_entry = audit_entries[1] + assert amendment_audit_entry.verb == "exporter_submitted_amendment" + assert amendment_audit_entry.target == original_application.case_ptr + assert amendment_audit_entry.payload == {"amendment": {"reference_code": amendment_application.reference_code}} class TestStandardApplication(DataTestClient): diff --git a/api/applications/views/applications.py b/api/applications/views/applications.py index 369454b1b3..b37ed75716 100644 --- a/api/applications/views/applications.py +++ b/api/applications/views/applications.py @@ -357,10 +357,10 @@ def put(self, request, pk): if application.case_type.sub_type == CaseTypeSubTypeEnum.STANDARD: set_case_flags_on_submitted_standard_application(application) + application.on_submit() add_goods_flags_to_submitted_application(application) apply_flagging_rules_to_case(application) - create_submitted_audit(request, application, old_status) auto_generate_case_document( "application_form", application, diff --git a/api/audit_trail/formatters.py b/api/audit_trail/formatters.py index f705bab928..c19ad0ef83 100644 --- a/api/audit_trail/formatters.py +++ b/api/audit_trail/formatters.py @@ -176,6 +176,8 @@ def upload_party_document(**payload): def get_updated_status(**payload): status = payload.get("status", "").lower() if status == CaseStatusEnum.SUBMITTED: + if payload.get("amendment_of"): + return f"submitted a 'major edit' to case {payload['amendment_of']['reference_code']}." return "applied for a licence." if status == CaseStatusEnum.RESUBMITTED: return "reapplied for a licence." From ce84afda8444f0fc8f1f35866506afb97d9ff24f Mon Sep 17 00:00:00 2001 From: Brendan Smith Date: Tue, 18 Jun 2024 16:56:13 +0100 Subject: [PATCH 32/73] Add audit entry to superseded case when amendment submitted --- .../libraries/application_helpers.py | 18 +- api/applications/models.py | 12 +- api/applications/views/applications.py | 2 +- api/audit_trail/enums.py | 1 + api/audit_trail/formatters.py | 4 + .../migrations/0024_alter_audit_verb.py | 278 ++++++++++++++++++ api/audit_trail/payload.py | 1 + 7 files changed, 296 insertions(+), 20 deletions(-) create mode 100644 api/audit_trail/migrations/0024_alter_audit_verb.py diff --git a/api/applications/libraries/application_helpers.py b/api/applications/libraries/application_helpers.py index ad9e765107..029b47055b 100644 --- a/api/applications/libraries/application_helpers.py +++ b/api/applications/libraries/application_helpers.py @@ -69,7 +69,7 @@ def can_status_be_set_by_gov_user(user: GovUser, original_status: str, new_statu def create_submitted_audit(user, application, old_status: str, additional_payload=None) -> None: - if not _additional_payload: + if not additional_payload: additional_payload = {} payload = { @@ -90,22 +90,6 @@ def create_submitted_audit(user, application, old_status: str, additional_payloa ) -def create_amendment_submitted_audit(user, application, old_status: str) -> None: - audit_trail_service.create( - actor=user, - verb=AuditType.UPDATED_STATUS, - target=application.get_case(), - payload={ - "status": { - "new": CaseStatusEnum.RESUBMITTED if old_status != CaseStatusEnum.DRAFT else CaseStatusEnum.SUBMITTED, - "old": old_status, - } - }, - ignore_case_status=True, - send_notification=False, - ) - - def check_user_can_set_status(request, application, data): """ Checks whether an user (internal/exporter) can set the requested status diff --git a/api/applications/models.py b/api/applications/models.py index cd42a10b82..aff1eef81b 100644 --- a/api/applications/models.py +++ b/api/applications/models.py @@ -201,11 +201,19 @@ class BaseApplication(ApplicationPartyMixin, Case): class Meta: ordering = ["created_at"] - def on_submit(self): + def on_submit(self, old_status): additional_payload = {} if self.amendment_of: + # Add an audit entry to the case that was superseded by this amendment + audit_trail_service.create_system_user_audit( + verb=AuditType.EXPORTER_SUBMITTED_AMENDMENT, + target=self.amendment_of, + payload={ + "amendment": {"reference_code": self.reference_code}, + }, + ) additional_payload["amendment_of"] = {"reference_code": self.amendment_of.reference_code} - create_submitted_audit(self.submitted_by, application, old_status, additional_payload) + create_submitted_audit(self.submitted_by, self, old_status, additional_payload) def add_to_queue(self, queue): case = self.get_case() diff --git a/api/applications/views/applications.py b/api/applications/views/applications.py index b37ed75716..525f5fa075 100644 --- a/api/applications/views/applications.py +++ b/api/applications/views/applications.py @@ -357,7 +357,7 @@ def put(self, request, pk): if application.case_type.sub_type == CaseTypeSubTypeEnum.STANDARD: set_case_flags_on_submitted_standard_application(application) - application.on_submit() + application.on_submit(old_status) add_goods_flags_to_submitted_application(application) apply_flagging_rules_to_case(application) diff --git a/api/audit_trail/enums.py b/api/audit_trail/enums.py index 6f280ef19b..5b8efd9b00 100644 --- a/api/audit_trail/enums.py +++ b/api/audit_trail/enums.py @@ -142,6 +142,7 @@ class AuditType(LiteEnum): CREATE_REFUSAL_CRITERIA = autostr() EXPORTER_APPEALED_REFUSAL = autostr() EXPORTER_CREATED_AMENDMENT = autostr() + EXPORTER_SUBMITTED_AMENDMENT = autostr() def human_readable(self): """ diff --git a/api/audit_trail/formatters.py b/api/audit_trail/formatters.py index c19ad0ef83..4354dc3627 100644 --- a/api/audit_trail/formatters.py +++ b/api/audit_trail/formatters.py @@ -341,3 +341,7 @@ def create_lu_meeting_note(advice_type, **payload): def create_refusal_criteria(**payload): return " added refusal criteria." + + +def exporter_submitted_amendment(**payload): + return f"created a new case for the edited application at {payload['amendment']['reference_code']}." diff --git a/api/audit_trail/migrations/0024_alter_audit_verb.py b/api/audit_trail/migrations/0024_alter_audit_verb.py new file mode 100644 index 0000000000..b27404b513 --- /dev/null +++ b/api/audit_trail/migrations/0024_alter_audit_verb.py @@ -0,0 +1,278 @@ +# Generated by Django 4.2.13 on 2024-06-18 15:18 + +import api.audit_trail.enums +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("audit_trail", "0023_alter_audit_verb"), + ] + + operations = [ + migrations.AlterField( + model_name="audit", + name="verb", + field=models.CharField( + choices=[ + (api.audit_trail.enums.AuditType["CREATED"], "created"), + (api.audit_trail.enums.AuditType["OGL_CREATED"], "ogl_created"), + (api.audit_trail.enums.AuditType["OGL_FIELD_EDITED"], "ogl_field_edited"), + (api.audit_trail.enums.AuditType["OGL_MULTI_FIELD_EDITED"], "ogl_multi_field_edited"), + (api.audit_trail.enums.AuditType["ADD_FLAGS"], "add_flags"), + (api.audit_trail.enums.AuditType["REMOVE_FLAGS"], "remove_flags"), + (api.audit_trail.enums.AuditType["GOOD_REVIEWED"], "good_reviewed"), + (api.audit_trail.enums.AuditType["GOOD_ADD_FLAGS"], "good_add_flags"), + (api.audit_trail.enums.AuditType["GOOD_REMOVE_FLAGS"], "good_remove_flags"), + (api.audit_trail.enums.AuditType["GOOD_ADD_REMOVE_FLAGS"], "good_add_remove_flags"), + (api.audit_trail.enums.AuditType["DESTINATION_ADD_FLAGS"], "destination_add_flags"), + (api.audit_trail.enums.AuditType["DESTINATION_REMOVE_FLAGS"], "destination_remove_flags"), + (api.audit_trail.enums.AuditType["ADD_GOOD_TO_APPLICATION"], "add_good_to_application"), + (api.audit_trail.enums.AuditType["REMOVE_GOOD_FROM_APPLICATION"], "remove_good_from_application"), + (api.audit_trail.enums.AuditType["ADD_GOOD_TYPE_TO_APPLICATION"], "add_good_type_to_application"), + ( + api.audit_trail.enums.AuditType["REMOVE_GOOD_TYPE_FROM_APPLICATION"], + "remove_good_type_from_application", + ), + ( + api.audit_trail.enums.AuditType["UPDATE_APPLICATION_END_USE_DETAIL"], + "update_application_end_use_detail", + ), + ( + api.audit_trail.enums.AuditType["UPDATE_APPLICATION_TEMPORARY_EXPORT"], + "update_application_temporary_export", + ), + ( + api.audit_trail.enums.AuditType["REMOVED_SITES_FROM_APPLICATION"], + "removed_sites_from_application", + ), + (api.audit_trail.enums.AuditType["ADD_SITES_TO_APPLICATION"], "add_sites_to_application"), + ( + api.audit_trail.enums.AuditType["REMOVED_EXTERNAL_LOCATIONS_FROM_APPLICATION"], + "removed_external_locations_from_application", + ), + ( + api.audit_trail.enums.AuditType["ADD_EXTERNAL_LOCATIONS_TO_APPLICATION"], + "add_external_locations_to_application", + ), + ( + api.audit_trail.enums.AuditType["REMOVED_COUNTRIES_FROM_APPLICATION"], + "removed_countries_from_application", + ), + (api.audit_trail.enums.AuditType["ADD_COUNTRIES_TO_APPLICATION"], "add_countries_to_application"), + ( + api.audit_trail.enums.AuditType["ADD_ADDITIONAL_CONTACT_TO_CASE"], + "add_additional_contact_to_case", + ), + (api.audit_trail.enums.AuditType["MOVE_CASE"], "move_case"), + (api.audit_trail.enums.AuditType["ASSIGN_CASE"], "assign_case"), + (api.audit_trail.enums.AuditType["ASSIGN_USER_TO_CASE"], "assign_user_to_case"), + (api.audit_trail.enums.AuditType["REMOVE_USER_FROM_CASE"], "remove_user_from_case"), + (api.audit_trail.enums.AuditType["REMOVE_CASE"], "remove_case"), + (api.audit_trail.enums.AuditType["REMOVE_CASE_FROM_ALL_QUEUES"], "remove_case_from_all_queues"), + ( + api.audit_trail.enums.AuditType["REMOVE_CASE_FROM_ALL_USER_ASSIGNMENTS"], + "remove_case_from_all_user_assignments", + ), + (api.audit_trail.enums.AuditType["CLC_RESPONSE"], "clc_response"), + (api.audit_trail.enums.AuditType["PV_GRADING_RESPONSE"], "pv_grading_response"), + (api.audit_trail.enums.AuditType["CREATED_CASE_NOTE"], "created_case_note"), + ( + api.audit_trail.enums.AuditType["CREATED_CASE_NOTE_WITH_MENTIONS"], + "created_case_note_with_mentions", + ), + (api.audit_trail.enums.AuditType["ECJU_QUERY"], "ecju_query"), + (api.audit_trail.enums.AuditType["ECJU_QUERY_RESPONSE"], "ecju_query_response"), + (api.audit_trail.enums.AuditType["ECJU_QUERY_MANUALLY_CLOSED"], "ecju_query_manually_closed"), + (api.audit_trail.enums.AuditType["UPDATED_STATUS"], "updated_status"), + (api.audit_trail.enums.AuditType["UPDATED_SUB_STATUS"], "updated_sub_status"), + (api.audit_trail.enums.AuditType["UPDATED_APPLICATION_NAME"], "updated_application_name"), + ( + api.audit_trail.enums.AuditType["UPDATE_APPLICATION_LETTER_REFERENCE"], + "update_application_letter_reference", + ), + ( + api.audit_trail.enums.AuditType["UPDATE_APPLICATION_F680_CLEARANCE_TYPES"], + "update_application_f680_clearance_types", + ), + ( + api.audit_trail.enums.AuditType["ADDED_APPLICATION_LETTER_REFERENCE"], + "added_application_letter_reference", + ), + ( + api.audit_trail.enums.AuditType["REMOVED_APPLICATION_LETTER_REFERENCE"], + "removed_application_letter_reference", + ), + (api.audit_trail.enums.AuditType["ASSIGNED_COUNTRIES_TO_GOOD"], "assigned_countries_to_good"), + (api.audit_trail.enums.AuditType["REMOVED_COUNTRIES_FROM_GOOD"], "removed_countries_from_good"), + (api.audit_trail.enums.AuditType["CREATED_FINAL_ADVICE"], "created_final_advice"), + (api.audit_trail.enums.AuditType["CLEARED_FINAL_ADVICE"], "cleared_final_advice"), + (api.audit_trail.enums.AuditType["CREATED_TEAM_ADVICE"], "created_team_advice"), + (api.audit_trail.enums.AuditType["CLEARED_TEAM_ADVICE"], "cleared_team_advice"), + (api.audit_trail.enums.AuditType["REVIEW_COMBINE_ADVICE"], "review_combine_advice"), + (api.audit_trail.enums.AuditType["CREATED_USER_ADVICE"], "created_user_advice"), + (api.audit_trail.enums.AuditType["CLEARED_USER_ADVICE"], "cleared_user_advice"), + (api.audit_trail.enums.AuditType["ADD_PARTY"], "add_party"), + (api.audit_trail.enums.AuditType["REMOVE_PARTY"], "remove_party"), + (api.audit_trail.enums.AuditType["UPLOAD_PARTY_DOCUMENT"], "upload_party_document"), + (api.audit_trail.enums.AuditType["DELETE_PARTY_DOCUMENT"], "delete_party_document"), + (api.audit_trail.enums.AuditType["UPLOAD_APPLICATION_DOCUMENT"], "upload_application_document"), + (api.audit_trail.enums.AuditType["DELETE_APPLICATION_DOCUMENT"], "delete_application_document"), + (api.audit_trail.enums.AuditType["UPLOAD_CASE_DOCUMENT"], "upload_case_document"), + (api.audit_trail.enums.AuditType["GENERATE_CASE_DOCUMENT"], "generate_case_document"), + (api.audit_trail.enums.AuditType["ADD_CASE_OFFICER_TO_CASE"], "add_case_officer_to_case"), + (api.audit_trail.enums.AuditType["REMOVE_CASE_OFFICER_FROM_CASE"], "remove_case_officer_from_case"), + (api.audit_trail.enums.AuditType["GRANTED_APPLICATION"], "granted_application"), + (api.audit_trail.enums.AuditType["REINSTATED_APPLICATION"], "reinstated_application"), + (api.audit_trail.enums.AuditType["FINALISED_APPLICATION"], "finalised_application"), + (api.audit_trail.enums.AuditType["UNASSIGNED_QUEUES"], "unassigned_queues"), + (api.audit_trail.enums.AuditType["UNASSIGNED"], "unassigned"), + (api.audit_trail.enums.AuditType["CREATED_DOCUMENT_TEMPLATE"], "created_document_template"), + (api.audit_trail.enums.AuditType["UPDATED_LETTER_TEMPLATE_NAME"], "updated_letter_template_name"), + ( + api.audit_trail.enums.AuditType["ADDED_LETTER_TEMPLATE_CASE_TYPES"], + "added_letter_template_case_types", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_LETTER_TEMPLATE_CASE_TYPES"], + "updated_letter_template_case_types", + ), + ( + api.audit_trail.enums.AuditType["REMOVED_LETTER_TEMPLATE_CASE_TYPES"], + "removed_letter_template_case_types", + ), + ( + api.audit_trail.enums.AuditType["ADDED_LETTER_TEMPLATE_DECISIONS"], + "added_letter_template_decisions", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_LETTER_TEMPLATE_DECISIONS"], + "updated_letter_template_decisions", + ), + ( + api.audit_trail.enums.AuditType["REMOVED_LETTER_TEMPLATE_DECISIONS"], + "removed_letter_template_decisions", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_LETTER_TEMPLATE_PARAGRAPHS"], + "updated_letter_template_paragraphs", + ), + ( + api.audit_trail.enums.AuditType["REMOVED_LETTER_TEMPLATE_PARAGRAPHS"], + "removed_letter_template_paragraphs", + ), + ( + api.audit_trail.enums.AuditType["ADDED_LETTER_TEMPLATE_PARAGRAPHS"], + "added_letter_template_paragraphs", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_LETTER_TEMPLATE_LAYOUT"], + "updated_letter_template_layout", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_LETTER_TEMPLATE_PARAGRAPHS_ORDERING"], + "updated_letter_template_paragraphs_ordering", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_LETTER_TEMPLATE_INCLUDE_DIGITAL_SIGNATURE"], + "updated_letter_template_include_digital_signature", + ), + (api.audit_trail.enums.AuditType["CREATED_PICKLIST"], "created_picklist"), + (api.audit_trail.enums.AuditType["UPDATED_PICKLIST_TEXT"], "updated_picklist_text"), + (api.audit_trail.enums.AuditType["UPDATED_PICKLIST_NAME"], "updated_picklist_name"), + (api.audit_trail.enums.AuditType["DEACTIVATE_PICKLIST"], "deactivate_picklist"), + (api.audit_trail.enums.AuditType["REACTIVATE_PICKLIST"], "reactivate_picklist"), + ( + api.audit_trail.enums.AuditType["UPDATED_EXHIBITION_DETAILS_TITLE"], + "updated_exhibition_details_title", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_EXHIBITION_DETAILS_START_DATE"], + "updated_exhibition_details_start_date", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_EXHIBITION_DETAILS_REQUIRED_BY_DATE"], + "updated_exhibition_details_required_by_date", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_EXHIBITION_DETAILS_REASON_FOR_CLEARANCE"], + "updated_exhibition_details_reason_for_clearance", + ), + (api.audit_trail.enums.AuditType["UPDATED_ROUTE_OF_GOODS"], "updated_route_of_goods"), + (api.audit_trail.enums.AuditType["UPDATED_ORGANISATION"], "updated_organisation"), + (api.audit_trail.enums.AuditType["CREATED_ORGANISATION"], "created_organisation"), + (api.audit_trail.enums.AuditType["REGISTER_ORGANISATION"], "register_organisation"), + (api.audit_trail.enums.AuditType["REJECTED_ORGANISATION"], "rejected_organisation"), + (api.audit_trail.enums.AuditType["APPROVED_ORGANISATION"], "approved_organisation"), + (api.audit_trail.enums.AuditType["REMOVED_FLAG_ON_ORGANISATION"], "removed_flag_on_organisation"), + (api.audit_trail.enums.AuditType["ADDED_FLAG_ON_ORGANISATION"], "added_flag_on_organisation"), + (api.audit_trail.enums.AuditType["RERUN_ROUTING_RULES"], "rerun_routing_rules"), + (api.audit_trail.enums.AuditType["ENFORCEMENT_CHECK"], "enforcement_check"), + (api.audit_trail.enums.AuditType["UPDATED_SITE"], "updated_site"), + (api.audit_trail.enums.AuditType["CREATED_SITE"], "created_site"), + (api.audit_trail.enums.AuditType["UPDATED_SITE_NAME"], "updated_site_name"), + (api.audit_trail.enums.AuditType["COMPLIANCE_SITE_CASE_CREATE"], "compliance_site_case_create"), + ( + api.audit_trail.enums.AuditType["COMPLIANCE_SITE_CASE_NEW_LICENCE"], + "compliance_site_case_new_licence", + ), + (api.audit_trail.enums.AuditType["ADDED_NEXT_REVIEW_DATE"], "added_next_review_date"), + (api.audit_trail.enums.AuditType["EDITED_NEXT_REVIEW_DATE"], "edited_next_review_date"), + (api.audit_trail.enums.AuditType["REMOVED_NEXT_REVIEW_DATE"], "removed_next_review_date"), + (api.audit_trail.enums.AuditType["COMPLIANCE_VISIT_CASE_CREATED"], "compliance_visit_case_created"), + (api.audit_trail.enums.AuditType["COMPLIANCE_VISIT_CASE_UPDATED"], "compliance_visit_case_updated"), + ( + api.audit_trail.enums.AuditType["COMPLIANCE_PEOPLE_PRESENT_CREATED"], + "compliance_people_present_created", + ), + ( + api.audit_trail.enums.AuditType["COMPLIANCE_PEOPLE_PRESENT_UPDATED"], + "compliance_people_present_updated", + ), + ( + api.audit_trail.enums.AuditType["COMPLIANCE_PEOPLE_PRESENT_DELETED"], + "compliance_people_present_deleted", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_GOOD_ON_DESTINATION_MATRIX"], + "updated_good_on_destination_matrix", + ), + (api.audit_trail.enums.AuditType["LICENCE_UPDATED_GOOD_USAGE"], "licence_updated_good_usage"), + (api.audit_trail.enums.AuditType["OGEL_REISSUED"], "ogel_reissued"), + (api.audit_trail.enums.AuditType["LICENCE_UPDATED_STATUS"], "licence_updated_status"), + ( + api.audit_trail.enums.AuditType["DOCUMENT_ON_ORGANISATION_CREATE"], + "document_on_organisation_create", + ), + ( + api.audit_trail.enums.AuditType["DOCUMENT_ON_ORGANISATION_DELETE"], + "document_on_organisation_delete", + ), + ( + api.audit_trail.enums.AuditType["DOCUMENT_ON_ORGANISATION_UPDATE"], + "document_on_organisation_update", + ), + (api.audit_trail.enums.AuditType["REPORT_SUMMARY_UPDATED"], "report_summary_updated"), + (api.audit_trail.enums.AuditType["COUNTERSIGN_ADVICE"], "countersign_advice"), + (api.audit_trail.enums.AuditType["UPDATED_SERIAL_NUMBERS"], "updated_serial_numbers"), + (api.audit_trail.enums.AuditType["PRODUCT_REVIEWED"], "product_reviewed"), + (api.audit_trail.enums.AuditType["LICENCE_UPDATED_PRODUCT_USAGE"], "licence_updated_product_usage"), + (api.audit_trail.enums.AuditType["CREATED_FINAL_RECOMMENDATION"], "created_final_recommendation"), + (api.audit_trail.enums.AuditType["GENERATE_DECISION_LETTER"], "generate_decision_letter"), + (api.audit_trail.enums.AuditType["DECISION_LETTER_SENT"], "decision_letter_sent"), + (api.audit_trail.enums.AuditType["LU_ADVICE"], "lu_advice"), + (api.audit_trail.enums.AuditType["LU_EDIT_ADVICE"], "lu_edit_advice"), + (api.audit_trail.enums.AuditType["LU_COUNTERSIGN"], "lu_countersign"), + (api.audit_trail.enums.AuditType["LU_EDIT_MEETING_NOTE"], "lu_edit_meeting_note"), + (api.audit_trail.enums.AuditType["LU_CREATE_MEETING_NOTE"], "lu_create_meeting_note"), + (api.audit_trail.enums.AuditType["CREATE_REFUSAL_CRITERIA"], "create_refusal_criteria"), + (api.audit_trail.enums.AuditType["EXPORTER_APPEALED_REFUSAL"], "exporter_appealed_refusal"), + (api.audit_trail.enums.AuditType["EXPORTER_CREATED_AMENDMENT"], "exporter_created_amendment"), + (api.audit_trail.enums.AuditType["EXPORTER_SUBMITTED_AMENDMENT"], "exporter_submitted_amendment"), + ], + db_index=True, + max_length=255, + ), + ), + ] diff --git a/api/audit_trail/payload.py b/api/audit_trail/payload.py index fcad7773da..e74dd5bd50 100644 --- a/api/audit_trail/payload.py +++ b/api/audit_trail/payload.py @@ -160,4 +160,5 @@ def format_payload(audit_type, payload): AuditType.CREATE_REFUSAL_CRITERIA: formatters.create_refusal_criteria, AuditType.EXPORTER_APPEALED_REFUSAL: " appealed refusal", AuditType.EXPORTER_CREATED_AMENDMENT: " opened the application for a 'major edit'", + AuditType.EXPORTER_SUBMITTED_AMENDMENT: formatters.exporter_submitted_amendment, } From f9fd8edbd10ab8fcdc68147e308528fa012595c7 Mon Sep 17 00:00:00 2001 From: Brendan Smith Date: Wed, 19 Jun 2024 11:42:21 +0100 Subject: [PATCH 33/73] Add audit entry on creation of amendment case --- api/applications/models.py | 6 + api/applications/tests/test_models.py | 8 +- api/audit_trail/enums.py | 1 + api/audit_trail/formatters.py | 4 + .../migrations/0025_alter_audit_verb.py | 279 ++++++++++++++++++ api/audit_trail/payload.py | 1 + 6 files changed, 298 insertions(+), 1 deletion(-) create mode 100644 api/audit_trail/migrations/0025_alter_audit_verb.py diff --git a/api/applications/models.py b/api/applications/models.py index aff1eef81b..aee7e6b724 100644 --- a/api/applications/models.py +++ b/api/applications/models.py @@ -384,6 +384,12 @@ def create_amendment(self, user): target=self.get_case(), payload={}, ) + audit_entry = audit_trail_service.create_system_user_audit( + verb=AuditType.AMENDMENT_CREATED, + target=amendment_application.case_ptr, + payload={"superseded_case": {"reference_code": self.reference_code}}, + ignore_case_status=True, + ) system_user = BaseUser.objects.get(id=SystemUser.id) self.case_ptr.change_status(system_user, get_case_status_by_status(CaseStatusEnum.SUPERSEDED_BY_AMENDMENT)) return amendment_application diff --git a/api/applications/tests/test_models.py b/api/applications/tests/test_models.py index cd6f90cae5..df5a806131 100644 --- a/api/applications/tests/test_models.py +++ b/api/applications/tests/test_models.py @@ -98,7 +98,13 @@ def test_create_amendment(self): assert original_application.status.status == "superseded_by_amendment" assert original_application.queues.all().count() == 0 audit_entries = Audit.objects.all() - amendment_audit_entry = audit_entries[1] + supersede_audit_entry = audit_entries[1] + assert supersede_audit_entry.payload == { + "superseded_case": {"reference_code": original_application.reference_code} + } + assert supersede_audit_entry.verb == "amendment_created" + assert supersede_audit_entry.target == amendment_application.case_ptr + amendment_audit_entry = audit_entries[2] assert amendment_audit_entry.payload == {} assert amendment_audit_entry.verb == "exporter_created_amendment" assert amendment_audit_entry.actor == exporter_user diff --git a/api/audit_trail/enums.py b/api/audit_trail/enums.py index 5b8efd9b00..e77d2af671 100644 --- a/api/audit_trail/enums.py +++ b/api/audit_trail/enums.py @@ -143,6 +143,7 @@ class AuditType(LiteEnum): EXPORTER_APPEALED_REFUSAL = autostr() EXPORTER_CREATED_AMENDMENT = autostr() EXPORTER_SUBMITTED_AMENDMENT = autostr() + AMENDMENT_CREATED = autostr() def human_readable(self): """ diff --git a/api/audit_trail/formatters.py b/api/audit_trail/formatters.py index 4354dc3627..1eebbc9c4a 100644 --- a/api/audit_trail/formatters.py +++ b/api/audit_trail/formatters.py @@ -345,3 +345,7 @@ def create_refusal_criteria(**payload): def exporter_submitted_amendment(**payload): return f"created a new case for the edited application at {payload['amendment']['reference_code']}." + + +def amendment_created(**payload): + return f"created the case to supersede {payload['superseded_case']['reference_code']}." diff --git a/api/audit_trail/migrations/0025_alter_audit_verb.py b/api/audit_trail/migrations/0025_alter_audit_verb.py new file mode 100644 index 0000000000..a10b8100d4 --- /dev/null +++ b/api/audit_trail/migrations/0025_alter_audit_verb.py @@ -0,0 +1,279 @@ +# Generated by Django 4.2.13 on 2024-06-19 09:30 + +import api.audit_trail.enums +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("audit_trail", "0024_alter_audit_verb"), + ] + + operations = [ + migrations.AlterField( + model_name="audit", + name="verb", + field=models.CharField( + choices=[ + (api.audit_trail.enums.AuditType["CREATED"], "created"), + (api.audit_trail.enums.AuditType["OGL_CREATED"], "ogl_created"), + (api.audit_trail.enums.AuditType["OGL_FIELD_EDITED"], "ogl_field_edited"), + (api.audit_trail.enums.AuditType["OGL_MULTI_FIELD_EDITED"], "ogl_multi_field_edited"), + (api.audit_trail.enums.AuditType["ADD_FLAGS"], "add_flags"), + (api.audit_trail.enums.AuditType["REMOVE_FLAGS"], "remove_flags"), + (api.audit_trail.enums.AuditType["GOOD_REVIEWED"], "good_reviewed"), + (api.audit_trail.enums.AuditType["GOOD_ADD_FLAGS"], "good_add_flags"), + (api.audit_trail.enums.AuditType["GOOD_REMOVE_FLAGS"], "good_remove_flags"), + (api.audit_trail.enums.AuditType["GOOD_ADD_REMOVE_FLAGS"], "good_add_remove_flags"), + (api.audit_trail.enums.AuditType["DESTINATION_ADD_FLAGS"], "destination_add_flags"), + (api.audit_trail.enums.AuditType["DESTINATION_REMOVE_FLAGS"], "destination_remove_flags"), + (api.audit_trail.enums.AuditType["ADD_GOOD_TO_APPLICATION"], "add_good_to_application"), + (api.audit_trail.enums.AuditType["REMOVE_GOOD_FROM_APPLICATION"], "remove_good_from_application"), + (api.audit_trail.enums.AuditType["ADD_GOOD_TYPE_TO_APPLICATION"], "add_good_type_to_application"), + ( + api.audit_trail.enums.AuditType["REMOVE_GOOD_TYPE_FROM_APPLICATION"], + "remove_good_type_from_application", + ), + ( + api.audit_trail.enums.AuditType["UPDATE_APPLICATION_END_USE_DETAIL"], + "update_application_end_use_detail", + ), + ( + api.audit_trail.enums.AuditType["UPDATE_APPLICATION_TEMPORARY_EXPORT"], + "update_application_temporary_export", + ), + ( + api.audit_trail.enums.AuditType["REMOVED_SITES_FROM_APPLICATION"], + "removed_sites_from_application", + ), + (api.audit_trail.enums.AuditType["ADD_SITES_TO_APPLICATION"], "add_sites_to_application"), + ( + api.audit_trail.enums.AuditType["REMOVED_EXTERNAL_LOCATIONS_FROM_APPLICATION"], + "removed_external_locations_from_application", + ), + ( + api.audit_trail.enums.AuditType["ADD_EXTERNAL_LOCATIONS_TO_APPLICATION"], + "add_external_locations_to_application", + ), + ( + api.audit_trail.enums.AuditType["REMOVED_COUNTRIES_FROM_APPLICATION"], + "removed_countries_from_application", + ), + (api.audit_trail.enums.AuditType["ADD_COUNTRIES_TO_APPLICATION"], "add_countries_to_application"), + ( + api.audit_trail.enums.AuditType["ADD_ADDITIONAL_CONTACT_TO_CASE"], + "add_additional_contact_to_case", + ), + (api.audit_trail.enums.AuditType["MOVE_CASE"], "move_case"), + (api.audit_trail.enums.AuditType["ASSIGN_CASE"], "assign_case"), + (api.audit_trail.enums.AuditType["ASSIGN_USER_TO_CASE"], "assign_user_to_case"), + (api.audit_trail.enums.AuditType["REMOVE_USER_FROM_CASE"], "remove_user_from_case"), + (api.audit_trail.enums.AuditType["REMOVE_CASE"], "remove_case"), + (api.audit_trail.enums.AuditType["REMOVE_CASE_FROM_ALL_QUEUES"], "remove_case_from_all_queues"), + ( + api.audit_trail.enums.AuditType["REMOVE_CASE_FROM_ALL_USER_ASSIGNMENTS"], + "remove_case_from_all_user_assignments", + ), + (api.audit_trail.enums.AuditType["CLC_RESPONSE"], "clc_response"), + (api.audit_trail.enums.AuditType["PV_GRADING_RESPONSE"], "pv_grading_response"), + (api.audit_trail.enums.AuditType["CREATED_CASE_NOTE"], "created_case_note"), + ( + api.audit_trail.enums.AuditType["CREATED_CASE_NOTE_WITH_MENTIONS"], + "created_case_note_with_mentions", + ), + (api.audit_trail.enums.AuditType["ECJU_QUERY"], "ecju_query"), + (api.audit_trail.enums.AuditType["ECJU_QUERY_RESPONSE"], "ecju_query_response"), + (api.audit_trail.enums.AuditType["ECJU_QUERY_MANUALLY_CLOSED"], "ecju_query_manually_closed"), + (api.audit_trail.enums.AuditType["UPDATED_STATUS"], "updated_status"), + (api.audit_trail.enums.AuditType["UPDATED_SUB_STATUS"], "updated_sub_status"), + (api.audit_trail.enums.AuditType["UPDATED_APPLICATION_NAME"], "updated_application_name"), + ( + api.audit_trail.enums.AuditType["UPDATE_APPLICATION_LETTER_REFERENCE"], + "update_application_letter_reference", + ), + ( + api.audit_trail.enums.AuditType["UPDATE_APPLICATION_F680_CLEARANCE_TYPES"], + "update_application_f680_clearance_types", + ), + ( + api.audit_trail.enums.AuditType["ADDED_APPLICATION_LETTER_REFERENCE"], + "added_application_letter_reference", + ), + ( + api.audit_trail.enums.AuditType["REMOVED_APPLICATION_LETTER_REFERENCE"], + "removed_application_letter_reference", + ), + (api.audit_trail.enums.AuditType["ASSIGNED_COUNTRIES_TO_GOOD"], "assigned_countries_to_good"), + (api.audit_trail.enums.AuditType["REMOVED_COUNTRIES_FROM_GOOD"], "removed_countries_from_good"), + (api.audit_trail.enums.AuditType["CREATED_FINAL_ADVICE"], "created_final_advice"), + (api.audit_trail.enums.AuditType["CLEARED_FINAL_ADVICE"], "cleared_final_advice"), + (api.audit_trail.enums.AuditType["CREATED_TEAM_ADVICE"], "created_team_advice"), + (api.audit_trail.enums.AuditType["CLEARED_TEAM_ADVICE"], "cleared_team_advice"), + (api.audit_trail.enums.AuditType["REVIEW_COMBINE_ADVICE"], "review_combine_advice"), + (api.audit_trail.enums.AuditType["CREATED_USER_ADVICE"], "created_user_advice"), + (api.audit_trail.enums.AuditType["CLEARED_USER_ADVICE"], "cleared_user_advice"), + (api.audit_trail.enums.AuditType["ADD_PARTY"], "add_party"), + (api.audit_trail.enums.AuditType["REMOVE_PARTY"], "remove_party"), + (api.audit_trail.enums.AuditType["UPLOAD_PARTY_DOCUMENT"], "upload_party_document"), + (api.audit_trail.enums.AuditType["DELETE_PARTY_DOCUMENT"], "delete_party_document"), + (api.audit_trail.enums.AuditType["UPLOAD_APPLICATION_DOCUMENT"], "upload_application_document"), + (api.audit_trail.enums.AuditType["DELETE_APPLICATION_DOCUMENT"], "delete_application_document"), + (api.audit_trail.enums.AuditType["UPLOAD_CASE_DOCUMENT"], "upload_case_document"), + (api.audit_trail.enums.AuditType["GENERATE_CASE_DOCUMENT"], "generate_case_document"), + (api.audit_trail.enums.AuditType["ADD_CASE_OFFICER_TO_CASE"], "add_case_officer_to_case"), + (api.audit_trail.enums.AuditType["REMOVE_CASE_OFFICER_FROM_CASE"], "remove_case_officer_from_case"), + (api.audit_trail.enums.AuditType["GRANTED_APPLICATION"], "granted_application"), + (api.audit_trail.enums.AuditType["REINSTATED_APPLICATION"], "reinstated_application"), + (api.audit_trail.enums.AuditType["FINALISED_APPLICATION"], "finalised_application"), + (api.audit_trail.enums.AuditType["UNASSIGNED_QUEUES"], "unassigned_queues"), + (api.audit_trail.enums.AuditType["UNASSIGNED"], "unassigned"), + (api.audit_trail.enums.AuditType["CREATED_DOCUMENT_TEMPLATE"], "created_document_template"), + (api.audit_trail.enums.AuditType["UPDATED_LETTER_TEMPLATE_NAME"], "updated_letter_template_name"), + ( + api.audit_trail.enums.AuditType["ADDED_LETTER_TEMPLATE_CASE_TYPES"], + "added_letter_template_case_types", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_LETTER_TEMPLATE_CASE_TYPES"], + "updated_letter_template_case_types", + ), + ( + api.audit_trail.enums.AuditType["REMOVED_LETTER_TEMPLATE_CASE_TYPES"], + "removed_letter_template_case_types", + ), + ( + api.audit_trail.enums.AuditType["ADDED_LETTER_TEMPLATE_DECISIONS"], + "added_letter_template_decisions", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_LETTER_TEMPLATE_DECISIONS"], + "updated_letter_template_decisions", + ), + ( + api.audit_trail.enums.AuditType["REMOVED_LETTER_TEMPLATE_DECISIONS"], + "removed_letter_template_decisions", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_LETTER_TEMPLATE_PARAGRAPHS"], + "updated_letter_template_paragraphs", + ), + ( + api.audit_trail.enums.AuditType["REMOVED_LETTER_TEMPLATE_PARAGRAPHS"], + "removed_letter_template_paragraphs", + ), + ( + api.audit_trail.enums.AuditType["ADDED_LETTER_TEMPLATE_PARAGRAPHS"], + "added_letter_template_paragraphs", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_LETTER_TEMPLATE_LAYOUT"], + "updated_letter_template_layout", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_LETTER_TEMPLATE_PARAGRAPHS_ORDERING"], + "updated_letter_template_paragraphs_ordering", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_LETTER_TEMPLATE_INCLUDE_DIGITAL_SIGNATURE"], + "updated_letter_template_include_digital_signature", + ), + (api.audit_trail.enums.AuditType["CREATED_PICKLIST"], "created_picklist"), + (api.audit_trail.enums.AuditType["UPDATED_PICKLIST_TEXT"], "updated_picklist_text"), + (api.audit_trail.enums.AuditType["UPDATED_PICKLIST_NAME"], "updated_picklist_name"), + (api.audit_trail.enums.AuditType["DEACTIVATE_PICKLIST"], "deactivate_picklist"), + (api.audit_trail.enums.AuditType["REACTIVATE_PICKLIST"], "reactivate_picklist"), + ( + api.audit_trail.enums.AuditType["UPDATED_EXHIBITION_DETAILS_TITLE"], + "updated_exhibition_details_title", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_EXHIBITION_DETAILS_START_DATE"], + "updated_exhibition_details_start_date", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_EXHIBITION_DETAILS_REQUIRED_BY_DATE"], + "updated_exhibition_details_required_by_date", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_EXHIBITION_DETAILS_REASON_FOR_CLEARANCE"], + "updated_exhibition_details_reason_for_clearance", + ), + (api.audit_trail.enums.AuditType["UPDATED_ROUTE_OF_GOODS"], "updated_route_of_goods"), + (api.audit_trail.enums.AuditType["UPDATED_ORGANISATION"], "updated_organisation"), + (api.audit_trail.enums.AuditType["CREATED_ORGANISATION"], "created_organisation"), + (api.audit_trail.enums.AuditType["REGISTER_ORGANISATION"], "register_organisation"), + (api.audit_trail.enums.AuditType["REJECTED_ORGANISATION"], "rejected_organisation"), + (api.audit_trail.enums.AuditType["APPROVED_ORGANISATION"], "approved_organisation"), + (api.audit_trail.enums.AuditType["REMOVED_FLAG_ON_ORGANISATION"], "removed_flag_on_organisation"), + (api.audit_trail.enums.AuditType["ADDED_FLAG_ON_ORGANISATION"], "added_flag_on_organisation"), + (api.audit_trail.enums.AuditType["RERUN_ROUTING_RULES"], "rerun_routing_rules"), + (api.audit_trail.enums.AuditType["ENFORCEMENT_CHECK"], "enforcement_check"), + (api.audit_trail.enums.AuditType["UPDATED_SITE"], "updated_site"), + (api.audit_trail.enums.AuditType["CREATED_SITE"], "created_site"), + (api.audit_trail.enums.AuditType["UPDATED_SITE_NAME"], "updated_site_name"), + (api.audit_trail.enums.AuditType["COMPLIANCE_SITE_CASE_CREATE"], "compliance_site_case_create"), + ( + api.audit_trail.enums.AuditType["COMPLIANCE_SITE_CASE_NEW_LICENCE"], + "compliance_site_case_new_licence", + ), + (api.audit_trail.enums.AuditType["ADDED_NEXT_REVIEW_DATE"], "added_next_review_date"), + (api.audit_trail.enums.AuditType["EDITED_NEXT_REVIEW_DATE"], "edited_next_review_date"), + (api.audit_trail.enums.AuditType["REMOVED_NEXT_REVIEW_DATE"], "removed_next_review_date"), + (api.audit_trail.enums.AuditType["COMPLIANCE_VISIT_CASE_CREATED"], "compliance_visit_case_created"), + (api.audit_trail.enums.AuditType["COMPLIANCE_VISIT_CASE_UPDATED"], "compliance_visit_case_updated"), + ( + api.audit_trail.enums.AuditType["COMPLIANCE_PEOPLE_PRESENT_CREATED"], + "compliance_people_present_created", + ), + ( + api.audit_trail.enums.AuditType["COMPLIANCE_PEOPLE_PRESENT_UPDATED"], + "compliance_people_present_updated", + ), + ( + api.audit_trail.enums.AuditType["COMPLIANCE_PEOPLE_PRESENT_DELETED"], + "compliance_people_present_deleted", + ), + ( + api.audit_trail.enums.AuditType["UPDATED_GOOD_ON_DESTINATION_MATRIX"], + "updated_good_on_destination_matrix", + ), + (api.audit_trail.enums.AuditType["LICENCE_UPDATED_GOOD_USAGE"], "licence_updated_good_usage"), + (api.audit_trail.enums.AuditType["OGEL_REISSUED"], "ogel_reissued"), + (api.audit_trail.enums.AuditType["LICENCE_UPDATED_STATUS"], "licence_updated_status"), + ( + api.audit_trail.enums.AuditType["DOCUMENT_ON_ORGANISATION_CREATE"], + "document_on_organisation_create", + ), + ( + api.audit_trail.enums.AuditType["DOCUMENT_ON_ORGANISATION_DELETE"], + "document_on_organisation_delete", + ), + ( + api.audit_trail.enums.AuditType["DOCUMENT_ON_ORGANISATION_UPDATE"], + "document_on_organisation_update", + ), + (api.audit_trail.enums.AuditType["REPORT_SUMMARY_UPDATED"], "report_summary_updated"), + (api.audit_trail.enums.AuditType["COUNTERSIGN_ADVICE"], "countersign_advice"), + (api.audit_trail.enums.AuditType["UPDATED_SERIAL_NUMBERS"], "updated_serial_numbers"), + (api.audit_trail.enums.AuditType["PRODUCT_REVIEWED"], "product_reviewed"), + (api.audit_trail.enums.AuditType["LICENCE_UPDATED_PRODUCT_USAGE"], "licence_updated_product_usage"), + (api.audit_trail.enums.AuditType["CREATED_FINAL_RECOMMENDATION"], "created_final_recommendation"), + (api.audit_trail.enums.AuditType["GENERATE_DECISION_LETTER"], "generate_decision_letter"), + (api.audit_trail.enums.AuditType["DECISION_LETTER_SENT"], "decision_letter_sent"), + (api.audit_trail.enums.AuditType["LU_ADVICE"], "lu_advice"), + (api.audit_trail.enums.AuditType["LU_EDIT_ADVICE"], "lu_edit_advice"), + (api.audit_trail.enums.AuditType["LU_COUNTERSIGN"], "lu_countersign"), + (api.audit_trail.enums.AuditType["LU_EDIT_MEETING_NOTE"], "lu_edit_meeting_note"), + (api.audit_trail.enums.AuditType["LU_CREATE_MEETING_NOTE"], "lu_create_meeting_note"), + (api.audit_trail.enums.AuditType["CREATE_REFUSAL_CRITERIA"], "create_refusal_criteria"), + (api.audit_trail.enums.AuditType["EXPORTER_APPEALED_REFUSAL"], "exporter_appealed_refusal"), + (api.audit_trail.enums.AuditType["EXPORTER_CREATED_AMENDMENT"], "exporter_created_amendment"), + (api.audit_trail.enums.AuditType["EXPORTER_SUBMITTED_AMENDMENT"], "exporter_submitted_amendment"), + (api.audit_trail.enums.AuditType["AMENDMENT_CREATED"], "amendment_created"), + ], + db_index=True, + max_length=255, + ), + ), + ] diff --git a/api/audit_trail/payload.py b/api/audit_trail/payload.py index e74dd5bd50..8cffd01050 100644 --- a/api/audit_trail/payload.py +++ b/api/audit_trail/payload.py @@ -161,4 +161,5 @@ def format_payload(audit_type, payload): AuditType.EXPORTER_APPEALED_REFUSAL: " appealed refusal", AuditType.EXPORTER_CREATED_AMENDMENT: " opened the application for a 'major edit'", AuditType.EXPORTER_SUBMITTED_AMENDMENT: formatters.exporter_submitted_amendment, + AuditType.AMENDMENT_CREATED: formatters.amendment_created, } From ccc131a194f05938195763fc828619297c278bad Mon Sep 17 00:00:00 2001 From: Brendan Smith Date: Wed, 19 Jun 2024 14:31:16 +0100 Subject: [PATCH 34/73] Fix lint errors --- api/applications/libraries/application_helpers.py | 1 - api/applications/models.py | 2 +- api/cases/models.py | 5 ----- api/cases/views/views.py | 4 ++-- 4 files changed, 3 insertions(+), 9 deletions(-) diff --git a/api/applications/libraries/application_helpers.py b/api/applications/libraries/application_helpers.py index 029b47055b..f87b229c4a 100644 --- a/api/applications/libraries/application_helpers.py +++ b/api/applications/libraries/application_helpers.py @@ -1,7 +1,6 @@ from django.http import JsonResponse from rest_framework import status -from rest_framework.request import Request from rest_framework.exceptions import PermissionDenied from api.audit_trail import service as audit_trail_service diff --git a/api/applications/models.py b/api/applications/models.py index aee7e6b724..7117fee1a2 100644 --- a/api/applications/models.py +++ b/api/applications/models.py @@ -384,7 +384,7 @@ def create_amendment(self, user): target=self.get_case(), payload={}, ) - audit_entry = audit_trail_service.create_system_user_audit( + audit_trail_service.create_system_user_audit( verb=AuditType.AMENDMENT_CREATED, target=amendment_application.case_ptr, payload={"superseded_case": {"reference_code": self.reference_code}}, diff --git a/api/cases/models.py b/api/cases/models.py index bae1c7cccd..e72dcede35 100644 --- a/api/cases/models.py +++ b/api/cases/models.py @@ -10,7 +10,6 @@ from django.utils import timezone from api.users.enums import UserType -from rest_framework.exceptions import ValidationError from queryable_properties.managers import QueryablePropertiesManager from queryable_properties.properties import queryable_property @@ -29,8 +28,6 @@ from api.cases.libraries.reference_code import generate_reference_code from api.cases.managers import CaseManager, CaseReferenceCodeManager, AdviceManager from api.common.models import TimestampableModel -from api.core.constants import GovPermissions -from api.core.permissions import assert_user_has_permission from api.documents.models import Document from api.flags.models import Flag from api.goods.enums import PvGrading @@ -45,7 +42,6 @@ CaseSubStatus, ) from api.teams.models import Team, Department -from api.users.enums import SystemUser from api.users.models import ( BaseUser, ExporterUser, @@ -53,7 +49,6 @@ UserOrganisationRelationship, ExporterNotification, ) -from lite_content.lite_api import strings denial_reasons_logger = logging.getLogger(settings.DENIAL_REASONS_DELETION_LOGGER) diff --git a/api/cases/views/views.py b/api/cases/views/views.py index ac30c83299..8e98572ba5 100644 --- a/api/cases/views/views.py +++ b/api/cases/views/views.py @@ -6,7 +6,7 @@ from django.contrib.contenttypes.models import ContentType from rest_framework import status -from rest_framework.exceptions import ParseError +from rest_framework.exceptions import ParseError, ValidationError from rest_framework.generics import ListCreateAPIView, UpdateAPIView, ListAPIView, RetrieveAPIView from rest_framework.views import APIView @@ -191,7 +191,7 @@ def patch(self, request, pk): # Only allow the final decision if the user has the MANAGE_FINAL_ADVICE permission if new_status.status == CaseStatusEnum.FINALISED: - assert_user_has_permission(user.govuser, GovPermissions.MANAGE_LICENCE_FINAL_ADVICE) + assert_user_has_permission(request.user.govuser, GovPermissions.MANAGE_LICENCE_FINAL_ADVICE) if not can_set_status(case, new_status.status): raise ValidationError({"status": [strings.Statuses.BAD_STATUS]}) From 259b35bfe032fb28921d6483cfda2cc8ee244448 Mon Sep 17 00:00:00 2001 From: Brendan Smith Date: Wed, 19 Jun 2024 15:31:46 +0100 Subject: [PATCH 35/73] Improve test coverage and drop redundant code --- api/audit_trail/tests/test_formatters.py | 10 ++++++++ api/cases/helpers.py | 29 ------------------------ api/cases/tests/test_status.py | 19 ++++++++++++++++ api/cases/views/views.py | 9 +------- 4 files changed, 30 insertions(+), 37 deletions(-) diff --git a/api/audit_trail/tests/test_formatters.py b/api/audit_trail/tests/test_formatters.py index 565aedbe99..089448604f 100644 --- a/api/audit_trail/tests/test_formatters.py +++ b/api/audit_trail/tests/test_formatters.py @@ -574,3 +574,13 @@ def test_update_lu_meeting_note(self, advice_status, expected_text): def test_create_lu_meeting_note(self, advice_status, expected_text): result = formatters.create_lu_meeting_note(advice_status) assert result == expected_text + + def test_exporter_submitted_amendment(self): + payload = {"amendment": {"reference_code": "some-ref"}} + result = formatters.exporter_submitted_amendment(**payload) + assert result == "created a new case for the edited application at some-ref." + + def test_amendment_created(self): + payload = {"superseded_case": {"reference_code": "some-ref"}} + result = formatters.amendment_created(**payload) + assert result == "created the case to supersede some-ref." diff --git a/api/cases/helpers.py b/api/cases/helpers.py index d06e80f799..d29770c6de 100644 --- a/api/cases/helpers.py +++ b/api/cases/helpers.py @@ -35,35 +35,6 @@ def get_updated_case_ids(user: GovUser): return GovNotification.objects.filter(user_id=user.pk, case__id__in=cases).values_list("case__id", flat=True) -def can_set_status(case, status): - """ - Returns true or false depending on different case conditions - """ - from api.compliance.models import ComplianceVisitCase - from api.compliance.helpers import compliance_visit_case_complete - - reference_type = case.case_type.reference - - if reference_type == CaseTypeReferenceEnum.COMP_SITE and status not in CaseStatusEnum.compliance_site_statuses: - return False - elif reference_type == CaseTypeReferenceEnum.COMP_VISIT and status not in CaseStatusEnum.compliance_visit_statuses: - return False - - if case.case_type.reference == CaseTypeReferenceEnum.COMP_VISIT and CaseStatusEnum.is_terminal(status): - comp_case = ComplianceVisitCase.objects.get(id=case.id) - if not compliance_visit_case_complete(comp_case): - return False - - if reference_type == CaseTypeReferenceEnum.CRE and status not in [ - CaseStatusEnum.CLOSED, - CaseStatusEnum.SUBMITTED, - CaseStatusEnum.RESUBMITTED, - ]: - return False - - return True - - def working_days_in_range(start_date, end_date): dates_in_range = [start_date + timedelta(n) for n in range((end_date - start_date).days)] return len([date for date in dates_in_range if (not is_bank_holiday(date) and not is_weekend(date))]) diff --git a/api/cases/tests/test_status.py b/api/cases/tests/test_status.py index 7f3a25b828..ac48b2fac2 100644 --- a/api/cases/tests/test_status.py +++ b/api/cases/tests/test_status.py @@ -62,6 +62,25 @@ def test_certain_case_statuses_changes_licence_status(self, case_status, licence self.assertEqual(self.case.status.status, case_status) self.assertEqual(licence.status, licence_status) + def test_change_status_no_user_permission(self): + data = {"status": CaseStatusEnum.FINALISED} + response = self.client.patch(self.url, data=data, **self.gov_headers) + + self.case.refresh_from_db() + + # This should be a 401, but legacy... + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + self.assertEqual(self.case.status.status, "submitted") + + def test_change_status_new_status_not_allowed(self): + data = {"status": CaseStatusEnum.APPLICANT_EDITING} + response = self.client.patch(self.url, data=data, **self.gov_headers) + + self.case.refresh_from_db() + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(self.case.status.status, "submitted") + # TODO: More tests covering different paths for status change view diff --git a/api/cases/views/views.py b/api/cases/views/views.py index 8e98572ba5..fa29e9f151 100644 --- a/api/cases/views/views.py +++ b/api/cases/views/views.py @@ -29,7 +29,7 @@ ) from api.cases.generated_documents.models import GeneratedCaseDocument from api.cases.generated_documents.serializers import AdviceDocumentGovSerializer -from api.cases.helpers import create_system_mention, can_set_status +from api.cases.helpers import create_system_mention from api.cases.libraries.advice import group_advice from api.cases.libraries.finalise import ( get_required_decision_document_types, @@ -189,13 +189,6 @@ def patch(self, request, pk): case = get_case(pk) new_status = get_case_status_by_status(request.data.get("status")) - # Only allow the final decision if the user has the MANAGE_FINAL_ADVICE permission - if new_status.status == CaseStatusEnum.FINALISED: - assert_user_has_permission(request.user.govuser, GovPermissions.MANAGE_LICENCE_FINAL_ADVICE) - - if not can_set_status(case, new_status.status): - raise ValidationError({"status": [strings.Statuses.BAD_STATUS]}) - if not can_status_be_set_by_gov_user(request.user.govuser, case.status.status, new_status.status, is_mod=False): raise ValidationError({"status": ["Status cannot be set by user"]}) From aec2fb9c4a969dfe81dc64212750fdb1c6fc3ad2 Mon Sep 17 00:00:00 2001 From: Brendan Smith Date: Wed, 19 Jun 2024 15:43:34 +0100 Subject: [PATCH 36/73] Fix lint complaints --- api/cases/helpers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/cases/helpers.py b/api/cases/helpers.py index d29770c6de..dd3854334e 100644 --- a/api/cases/helpers.py +++ b/api/cases/helpers.py @@ -2,8 +2,6 @@ from api.audit_trail.enums import AuditType from api.common.dates import is_bank_holiday, is_weekend -from api.cases.enums import CaseTypeReferenceEnum -from api.staticdata.statuses.enums import CaseStatusEnum from api.users.models import BaseUser, GovUser, GovNotification from api.users.enums import SystemUser From 6aa1eced6d70bf880b9235f34783ad9027534b1c Mon Sep 17 00:00:00 2001 From: Brendan Smith Date: Mon, 24 Jun 2024 17:23:42 +0100 Subject: [PATCH 37/73] Fix ruff complaint --- api/applications/tests/test_models.py | 3 --- api/cases/tests/test_models.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/api/applications/tests/test_models.py b/api/applications/tests/test_models.py index df5a806131..5d41fa7974 100644 --- a/api/applications/tests/test_models.py +++ b/api/applications/tests/test_models.py @@ -1,6 +1,5 @@ from django.forms import model_to_dict from django.utils import timezone -from freezegun import freeze_time from test_helpers.clients import DataTestClient @@ -9,7 +8,6 @@ from api.cases.models import CaseType, Queue from api.flags.models import Flag from api.applications.models import ( - ExternalLocationOnApplication, GoodOnApplication, GoodOnApplicationDocument, GoodOnApplicationInternalDocument, @@ -31,7 +29,6 @@ from api.goods.tests.factories import FirearmFactory from api.organisations.tests.factories import OrganisationFactory from api.staticdata.control_list_entries.models import ControlListEntry -from api.staticdata.regimes.models import RegimeEntry from api.staticdata.report_summaries.models import ReportSummary, ReportSummaryPrefix, ReportSummarySubject from api.staticdata.statuses.models import CaseStatus, CaseSubStatus from api.users.models import ExporterUser diff --git a/api/cases/tests/test_models.py b/api/cases/tests/test_models.py index f1e9b92cff..b0acf32ca8 100644 --- a/api/cases/tests/test_models.py +++ b/api/cases/tests/test_models.py @@ -4,7 +4,7 @@ from api.audit_trail.serializers import AuditSerializer from parameterized import parameterized -from api.cases.models import Case, BadSubStatus +from api.cases.models import BadSubStatus from api.cases.tests.factories import CaseFactory from api.staticdata.statuses.enums import CaseStatusEnum, CaseSubStatusIdEnum from api.staticdata.statuses.models import CaseStatus, CaseSubStatus From 9863286b934835450efc5a8cec8d5a1fd7cb53de Mon Sep 17 00:00:00 2001 From: Le H Ngo Date: Mon, 10 Jun 2024 12:48:40 +0100 Subject: [PATCH 38/73] Adds submitted date to serializer --- api/applications/serializers/generic_application.py | 1 + 1 file changed, 1 insertion(+) diff --git a/api/applications/serializers/generic_application.py b/api/applications/serializers/generic_application.py index 486383206d..e55fe8a85c 100644 --- a/api/applications/serializers/generic_application.py +++ b/api/applications/serializers/generic_application.py @@ -44,6 +44,7 @@ class GenericApplicationListSerializer(serializers.Serializer): name = serializers.CharField() case_type = TinyCaseTypeSerializer() status = CaseStatusField() + submitted_at = serializers.DateTimeField() updated_at = serializers.DateTimeField() reference_code = serializers.CharField() export_type = serializers.SerializerMethodField() From 29d1bd54e46a32c5d14584d5463a5d0458c9b204 Mon Sep 17 00:00:00 2001 From: Le H Ngo Date: Mon, 17 Jun 2024 11:00:06 +0100 Subject: [PATCH 39/73] Adds filter for finalised applications --- api/applications/managers.py | 4 ++++ api/applications/views/applications.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/api/applications/managers.py b/api/applications/managers.py index 0367c87bb3..abd87be0f8 100644 --- a/api/applications/managers.py +++ b/api/applications/managers.py @@ -12,3 +12,7 @@ def drafts(self, organisation): def submitted(self, organisation): draft = get_case_status_by_status(CaseStatusEnum.DRAFT) return self.get_queryset().filter(organisation=organisation).exclude(status=draft).order_by("-submitted_at") + + def finalised(self, organisation): + finalised = get_case_status_by_status(CaseStatusEnum.FINALISED) + return self.get_queryset().filter(status=finalised, organisation=organisation).order_by("-updated_at") diff --git a/api/applications/views/applications.py b/api/applications/views/applications.py index 525f5fa075..e5aa2d17a1 100644 --- a/api/applications/views/applications.py +++ b/api/applications/views/applications.py @@ -119,8 +119,12 @@ def get_queryset(self): organisation = get_request_user_organisation(self.request) + finalised = optional_str_to_bool(self.request.GET.get("finalised")) + if submitted is None: applications = BaseApplication.objects.filter(organisation=organisation) + elif submitted and finalised: + applications = BaseApplication.objects.finalised(organisation) elif submitted: applications = BaseApplication.objects.submitted(organisation) else: From 4d35d183214820cca8027a1c7f4d7de4be05c286 Mon Sep 17 00:00:00 2001 From: Le H Ngo Date: Wed, 19 Jun 2024 00:31:36 +0100 Subject: [PATCH 40/73] Adds sorting query --- api/applications/managers.py | 8 ++++---- api/applications/views/applications.py | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/api/applications/managers.py b/api/applications/managers.py index abd87be0f8..4f0723e364 100644 --- a/api/applications/managers.py +++ b/api/applications/managers.py @@ -9,10 +9,10 @@ def drafts(self, organisation): draft = get_case_status_by_status(CaseStatusEnum.DRAFT) return self.get_queryset().filter(status=draft, organisation=organisation).order_by("-created_at") - def submitted(self, organisation): + def submitted(self, organisation, sort): draft = get_case_status_by_status(CaseStatusEnum.DRAFT) - return self.get_queryset().filter(organisation=organisation).exclude(status=draft).order_by("-submitted_at") + return self.get_queryset().filter(organisation=organisation).exclude(status=draft).order_by(sort) - def finalised(self, organisation): + def finalised(self, organisation, sort): finalised = get_case_status_by_status(CaseStatusEnum.FINALISED) - return self.get_queryset().filter(status=finalised, organisation=organisation).order_by("-updated_at") + return self.get_queryset().filter(status=finalised, organisation=organisation).order_by(sort) diff --git a/api/applications/views/applications.py b/api/applications/views/applications.py index e5aa2d17a1..6e770d3242 100644 --- a/api/applications/views/applications.py +++ b/api/applications/views/applications.py @@ -120,13 +120,14 @@ def get_queryset(self): organisation = get_request_user_organisation(self.request) finalised = optional_str_to_bool(self.request.GET.get("finalised")) + sort = self.request.GET.get("sort", "-updated_at") if submitted is None: applications = BaseApplication.objects.filter(organisation=organisation) elif submitted and finalised: - applications = BaseApplication.objects.finalised(organisation) + applications = BaseApplication.objects.finalised(organisation, sort) elif submitted: - applications = BaseApplication.objects.submitted(organisation) + applications = BaseApplication.objects.submitted(organisation, sort) else: applications = BaseApplication.objects.drafts(organisation) From 4091bd67b5734d1bdd8f142f04a39abe253ab44a Mon Sep 17 00:00:00 2001 From: Le H Ngo Date: Wed, 19 Jun 2024 12:13:43 +0100 Subject: [PATCH 41/73] Adds unit test for the finalised filter --- .../tests/test_view_application.py | 58 ++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/api/applications/tests/test_view_application.py b/api/applications/tests/test_view_application.py index e1e17d962f..6b34f13cbf 100644 --- a/api/applications/tests/test_view_application.py +++ b/api/applications/tests/test_view_application.py @@ -1,16 +1,23 @@ from uuid import UUID from django.urls import reverse +from api.cases.tests.factories import FinalAdviceFactory +from api.licences.enums import LicenceStatus +from api.licences.tests.factories import StandardLicenceFactory +from api.staticdata.decisions.models import Decision +from api.staticdata.statuses.models import CaseStatus from parameterized import parameterized from rest_framework import status from api.applications.models import GoodOnApplication, SiteOnApplication -from api.cases.enums import CaseTypeEnum +from api.cases.enums import AdviceType, CaseTypeEnum from api.organisations.tests.factories import SiteFactory from api.staticdata.statuses.enums import CaseStatusEnum from api.staticdata.trade_control.enums import TradeControlActivity, TradeControlProductCategory from test_helpers.clients import DataTestClient from api.users.libraries.get_user import get_user_organisation_relationship +from api.staticdata.statuses.libraries.get_case_status import get_case_status_by_status +from api.core.constants import GovPermissions class DraftTests(DataTestClient): @@ -203,3 +210,52 @@ def test_organisation_has_existing_applications(self): response = self.client.get(url, **self.exporter_headers) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.json()["applications"], True) + + def test_view_finalised_applications(self): + url = reverse("applications:applications") + "?submitted=true" + response = self.client.get(url, **self.exporter_headers) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.json()["count"], 0) + + self.exporter_user.set_role(self.organisation, self.exporter_super_user_role) + application = self.create_draft_standard_application(self.organisation) + + self.submit_application(application) + url = reverse("applications:applications") + response = self.client.get(url, **self.exporter_headers) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.json()["count"], 1) + + application_finalised = self.create_standard_application_case(self.organisation) + FinalAdviceFactory(user=self.gov_user, case=application_finalised, type=AdviceType.APPROVE) + template = self.create_letter_template( + name="Template", + case_types=[CaseTypeEnum.SIEL.id], + decisions=[Decision.objects.get(name=AdviceType.APPROVE)], + ) + + self.gov_user.role.permissions.set([GovPermissions.MANAGE_LICENCE_FINAL_ADVICE.name]) + licence = StandardLicenceFactory(case=application_finalised, status=LicenceStatus.DRAFT) + self.create_generated_case_document( + application_finalised, template, advice_type=AdviceType.APPROVE, licence=licence + ) + + finalised_url = reverse("cases:finalise", kwargs={"pk": application_finalised.id}) + response = self.client.put(finalised_url, data={}, **self.gov_headers) + application_finalised.refresh_from_db() + + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(application_finalised.status, CaseStatus.objects.get(status=CaseStatusEnum.FINALISED)) + + response = self.client.get(url, **self.exporter_headers) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.json()["count"], 2) + + url = reverse("applications:applications") + "?submitted=true&finalised=true" + response = self.client.get(url, **self.exporter_headers) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.json()["count"], 1) + self.assertEqual( + response.json()["results"][0]["status"]["value"], + "Finalised", + ) From 79c79079cf0319db4d3a5d3e5a7c4cbb67391a89 Mon Sep 17 00:00:00 2001 From: Le H Ngo Date: Thu, 20 Jun 2024 13:09:44 +0100 Subject: [PATCH 42/73] Additional test for sort querystring --- .../tests/test_view_application.py | 30 +++++++++++++++++-- api/applications/views/applications.py | 13 ++++---- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/api/applications/tests/test_view_application.py b/api/applications/tests/test_view_application.py index 6b34f13cbf..37680e3997 100644 --- a/api/applications/tests/test_view_application.py +++ b/api/applications/tests/test_view_application.py @@ -1,5 +1,7 @@ from uuid import UUID +import datetime +from django.utils import timezone from django.urls import reverse from api.cases.tests.factories import FinalAdviceFactory from api.licences.enums import LicenceStatus @@ -16,7 +18,6 @@ from api.staticdata.trade_control.enums import TradeControlActivity, TradeControlProductCategory from test_helpers.clients import DataTestClient from api.users.libraries.get_user import get_user_organisation_relationship -from api.staticdata.statuses.libraries.get_case_status import get_case_status_by_status from api.core.constants import GovPermissions @@ -221,7 +222,7 @@ def test_view_finalised_applications(self): application = self.create_draft_standard_application(self.organisation) self.submit_application(application) - url = reverse("applications:applications") + url = reverse("applications:applications") + "?sort=submitted_at" response = self.client.get(url, **self.exporter_headers) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.json()["count"], 1) @@ -242,6 +243,9 @@ def test_view_finalised_applications(self): finalised_url = reverse("cases:finalise", kwargs={"pk": application_finalised.id}) response = self.client.put(finalised_url, data={}, **self.gov_headers) + + application.submitted_at = timezone.make_aware(datetime.datetime(2020, 6, 20, 12, 0)) + application.save() application_finalised.refresh_from_db() self.assertEqual(response.status_code, status.HTTP_201_CREATED) @@ -251,6 +255,28 @@ def test_view_finalised_applications(self): self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.json()["count"], 2) + data = response.json() + submitted_dates = [ + datetime.datetime.fromisoformat(item["submitted_at"].rstrip("Z").replace("Z", "+00:00")) + for item in data["results"] + ] + assert all( + submitted_dates[i] <= submitted_dates[i + 1] for i in range(len(submitted_dates) - 1) + ), "Dates are not in ascending order." + + url = reverse("applications:applications") + "?sort=-updated_at" + response = self.client.get(url, **self.exporter_headers) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + data = response.json() + updated_dates = [ + datetime.datetime.fromisoformat(item["updated_at"].rstrip("Z").replace("Z", "+00:00")) + for item in data["results"] + ] + assert all( + updated_dates[i] >= updated_dates[i + 1] for i in range(len(updated_dates) - 1) + ), "Dates are not in descending order." + url = reverse("applications:applications") + "?submitted=true&finalised=true" response = self.client.get(url, **self.exporter_headers) self.assertEqual(response.status_code, status.HTTP_200_OK) diff --git a/api/applications/views/applications.py b/api/applications/views/applications.py index 6e770d3242..9cc7de6db0 100644 --- a/api/applications/views/applications.py +++ b/api/applications/views/applications.py @@ -113,20 +113,19 @@ def get_queryset(self): Filter applications on submitted """ try: - submitted = optional_str_to_bool(self.request.GET.get("submitted")) + submitted_tab = optional_str_to_bool(self.request.GET.get("submitted")) except ValueError: return BaseApplication.objects.none() organisation = get_request_user_organisation(self.request) + finalised_tab = optional_str_to_bool(self.request.GET.get("finalised")) + sort = self.request.GET.get("sort", "submitted_at") - finalised = optional_str_to_bool(self.request.GET.get("finalised")) - sort = self.request.GET.get("sort", "-updated_at") - - if submitted is None: + if submitted_tab is None: applications = BaseApplication.objects.filter(organisation=organisation) - elif submitted and finalised: + elif finalised_tab: applications = BaseApplication.objects.finalised(organisation, sort) - elif submitted: + elif submitted_tab: applications = BaseApplication.objects.submitted(organisation, sort) else: applications = BaseApplication.objects.drafts(organisation) From a25980117d61b3feecbd204ca1199463b4745033 Mon Sep 17 00:00:00 2001 From: Le H Ngo Date: Fri, 21 Jun 2024 12:17:55 +0100 Subject: [PATCH 43/73] Small refactor --- api/applications/managers.py | 12 ++++----- .../tests/test_view_application.py | 10 ++++---- api/applications/views/applications.py | 25 +++++++------------ 3 files changed, 20 insertions(+), 27 deletions(-) diff --git a/api/applications/managers.py b/api/applications/managers.py index 4f0723e364..fdb2059dde 100644 --- a/api/applications/managers.py +++ b/api/applications/managers.py @@ -5,14 +5,14 @@ class BaseApplicationManager(InheritanceManager): - def drafts(self, organisation): + def drafts(self, organisation, sort_by): draft = get_case_status_by_status(CaseStatusEnum.DRAFT) - return self.get_queryset().filter(status=draft, organisation=organisation).order_by("-created_at") + return self.get_queryset().filter(status=draft, organisation=organisation).order_by(sort_by) - def submitted(self, organisation, sort): + def submitted(self, organisation, sort_by): draft = get_case_status_by_status(CaseStatusEnum.DRAFT) - return self.get_queryset().filter(organisation=organisation).exclude(status=draft).order_by(sort) + return self.get_queryset().filter(organisation=organisation).exclude(status=draft).order_by(sort_by) - def finalised(self, organisation, sort): + def finalised(self, organisation, sort_by): finalised = get_case_status_by_status(CaseStatusEnum.FINALISED) - return self.get_queryset().filter(status=finalised, organisation=organisation).order_by(sort) + return self.get_queryset().filter(status=finalised, organisation=organisation).order_by(sort_by) diff --git a/api/applications/tests/test_view_application.py b/api/applications/tests/test_view_application.py index 37680e3997..75b7b74f04 100644 --- a/api/applications/tests/test_view_application.py +++ b/api/applications/tests/test_view_application.py @@ -24,7 +24,7 @@ class DraftTests(DataTestClient): def setUp(self): super().setUp() - self.url = reverse("applications:applications") + "?submitted=false" + self.url = reverse("applications:applications") + "?sort_by=-created_at&selected_filter=draft_applications" def test_view_draft_standard_application_list_as_exporter_success(self): """ @@ -213,7 +213,7 @@ def test_organisation_has_existing_applications(self): self.assertEqual(response.json()["applications"], True) def test_view_finalised_applications(self): - url = reverse("applications:applications") + "?submitted=true" + url = reverse("applications:applications") response = self.client.get(url, **self.exporter_headers) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.json()["count"], 0) @@ -222,7 +222,7 @@ def test_view_finalised_applications(self): application = self.create_draft_standard_application(self.organisation) self.submit_application(application) - url = reverse("applications:applications") + "?sort=submitted_at" + url = reverse("applications:applications") + "?sort_by=submitted_at" response = self.client.get(url, **self.exporter_headers) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.json()["count"], 1) @@ -264,7 +264,7 @@ def test_view_finalised_applications(self): submitted_dates[i] <= submitted_dates[i + 1] for i in range(len(submitted_dates) - 1) ), "Dates are not in ascending order." - url = reverse("applications:applications") + "?sort=-updated_at" + url = reverse("applications:applications") + "?sort_by=-updated_at" response = self.client.get(url, **self.exporter_headers) self.assertEqual(response.status_code, status.HTTP_200_OK) @@ -277,7 +277,7 @@ def test_view_finalised_applications(self): updated_dates[i] >= updated_dates[i + 1] for i in range(len(updated_dates) - 1) ), "Dates are not in descending order." - url = reverse("applications:applications") + "?submitted=true&finalised=true" + url = reverse("applications:applications") + "?selected_filter=finalised_applications" response = self.client.get(url, **self.exporter_headers) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.json()["count"], 1) diff --git a/api/applications/views/applications.py b/api/applications/views/applications.py index 9cc7de6db0..4808ebe47c 100644 --- a/api/applications/views/applications.py +++ b/api/applications/views/applications.py @@ -31,7 +31,6 @@ auto_match_sanctions, ) from api.applications.libraries.application_helpers import ( - optional_str_to_bool, can_status_be_set_by_gov_user, create_submitted_audit, check_user_can_set_status, @@ -112,23 +111,17 @@ def get_queryset(self): """ Filter applications on submitted """ - try: - submitted_tab = optional_str_to_bool(self.request.GET.get("submitted")) - except ValueError: - return BaseApplication.objects.none() - organisation = get_request_user_organisation(self.request) - finalised_tab = optional_str_to_bool(self.request.GET.get("finalised")) - sort = self.request.GET.get("sort", "submitted_at") - - if submitted_tab is None: - applications = BaseApplication.objects.filter(organisation=organisation) - elif finalised_tab: - applications = BaseApplication.objects.finalised(organisation, sort) - elif submitted_tab: - applications = BaseApplication.objects.submitted(organisation, sort) + + selected_filter = self.request.GET.get("selected_filter", "submitted_applications") + sort_by = self.request.GET.get("sort_by", "-submitted_at") + + if selected_filter == "submitted_applications": + applications = BaseApplication.objects.submitted(organisation, sort_by) + elif selected_filter == "finalised_applications": + applications = BaseApplication.objects.finalised(organisation, sort_by) else: - applications = BaseApplication.objects.drafts(organisation) + applications = BaseApplication.objects.drafts(organisation, sort_by) users_sites = Site.objects.get_by_user_and_organisation(self.request.user.exporteruser, organisation) disallowed_applications = SiteOnApplication.objects.exclude(site__id__in=users_sites).values_list( From ac91110bf9c85812cd9f66e1ad6a9339cafc7e33 Mon Sep 17 00:00:00 2001 From: Gurdeep Atwal Date: Mon, 24 Jun 2024 12:12:41 +0100 Subject: [PATCH 44/73] application settings --- Pipfile | 1 + Pipfile.lock | 54 +++++---- api/conf/celery.py | 5 +- api/conf/settings.py | 270 +++++++++++++++++++++++++++---------------- 4 files changed, 203 insertions(+), 127 deletions(-) diff --git a/Pipfile b/Pipfile index f64868c9cb..9e198d4261 100644 --- a/Pipfile +++ b/Pipfile @@ -75,6 +75,7 @@ django-reversion = ">=5.0.12" psycopg = "~=3.1.18" django-log-formatter-asim = "~=0.0.5" dbt-copilot-python = "~=0.2.1" +dj-database-url = "~=2.2.0" [requires] python_version = "3.9" diff --git a/Pipfile.lock b/Pipfile.lock index 860ef4c9bf..317598a205 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "792919a16ef11b6fadff4d71b4f1b37f6b05115c9f69b01b4d5fed61dab00add" + "sha256": "c6c946c69f5bee64c779b1c82f29cecbfe5c23d1c819e5e041d387e221a66086" }, "pipfile-spec": 6, "requires": { @@ -43,11 +43,11 @@ }, "autopep8": { "hashes": [ - "sha256:5cfe45eb3bef8662f6a3c7e28b7c0310c7310d340074b7f0f28f9810b44b7ef4", - "sha256:b716efa70cbafbf4a2c9c5ec1cabfa037a68f9e30b04c74ffa5864dd49b8f7d2" + "sha256:8d6c87eba648fdcfc83e29b788910b8643171c395d9c4bcf115ece035b9c9dda", + "sha256:a203fe0fcad7939987422140ab17a930f684763bf7335bdb6709991dd7ef6c2d" ], "markers": "python_version >= '3.8'", - "version": "==2.3.0" + "version": "==2.3.1" }, "backcall": { "hashes": [ @@ -419,6 +419,14 @@ ], "version": "==0.3.8" }, + "dj-database-url": { + "hashes": [ + "sha256:3e792567b0aa9a4884860af05fe2aa4968071ad351e033b6db632f97ac6db9de", + "sha256:9f9b05058ddf888f1e6f840048b8d705ff9395e3b52a07165daa3d8b9360551b" + ], + "index": "pypi", + "version": "==2.2.0" + }, "django": { "hashes": [ "sha256:837e3cf1f6c31347a1396a3f6b65688f2b4bb4a11c580dcb628b5afe527b68a5", @@ -490,12 +498,12 @@ }, "django-health-check": { "hashes": [ - "sha256:16f9c9186236cbc2858fa0d0ecc3566ba2ad2b72683e5678d0d58eb9e8bbba1a", - "sha256:21235120f8d756fa75ba430d0b0dbb04620fbd7bfac92ed6a0b911915ba38918" + "sha256:18b75daca4551c69a43f804f9e41e23f5f5fb9efd06cf6a313b3d5031bb87bd0", + "sha256:f5f58762b80bdf7b12fad724761993d6e83540f97e2c95c42978f187e452fa07" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==3.18.2" + "version": "==3.18.3" }, "django-ipware": { "hashes": [ @@ -695,18 +703,18 @@ }, "filelock": { "hashes": [ - "sha256:0151273e5b5d6cf753a61ec83b3a9b7d8821c39ae9af9d7ecf2f9e2f17404103", - "sha256:e1199bf5194a2277273dacd50269f0d87d0682088a3c561c15674ea9005d8635" + "sha256:2207938cbc1844345cb01a5a95524dae30f0ce089eba5b00378295a17e3e90cb", + "sha256:6ca1fffae96225dab4c6eaf1c4f4f28cd2568d3ec2a44e15a08520504de468e7" ], "markers": "python_version >= '3.8'", - "version": "==3.15.3" + "version": "==3.15.4" }, "future": { "hashes": [ "sha256:929292d34f5872e70396626ef385ec22355a1fae8ad29e1a734c3e43f9fbc216", "sha256:bd2968309307861edae1458a4f8a4f3598c03be43b97521076aebf5d94c07b05" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.0.0" }, "gevent": { @@ -833,7 +841,7 @@ "sha256:fd096eb7ffef17c456cfa587523c5f92321ae02427ff955bebe9e3c63bc9f0da", "sha256:fe754d231288e1e64323cfad462fcee8f0288654c10bdf4f603a39ed923bef33" ], - "markers": "platform_python_implementation == 'CPython' and python_version < '3.11'", + "markers": "python_version < '3.11' and platform_python_implementation == 'CPython'", "version": "==3.0.3" }, "grpcio": { @@ -1386,7 +1394,7 @@ "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.9.0.post0" }, "pytz": { @@ -1516,7 +1524,7 @@ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, "sqlparse": { @@ -1589,7 +1597,7 @@ "sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3", "sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429" ], - "markers": "python_version >= '3.6'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", "version": "==1.26.19" }, "vine": { @@ -1602,11 +1610,11 @@ }, "virtualenv": { "hashes": [ - "sha256:82bf0f4eebbb78d36ddaee0283d43fe5736b53880b8a8cdcd37390a07ac3741c", - "sha256:a624db5e94f01ad993d476b9ee5346fdf7b9de43ccaee0e0197012dc838a0e9b" + "sha256:4c43a2a236279d9ea36a0d76f98d84bd6ca94ac4e0f4a3b9d46d05e10fea542a", + "sha256:8cc4a31139e796e9a7de2cd5cf2489de1217193116a8fd42328f1bd65f434589" ], "markers": "python_version >= '3.7'", - "version": "==20.26.2" + "version": "==20.26.3" }, "wcwidth": { "hashes": [ @@ -2424,7 +2432,7 @@ "sha256:06d39a8b70fde873eb2a131141a0e79bb34a432941fb3d66fad247abafc9766c", "sha256:79b1f2497060d0928bc46016793f1fca1057c4aacdf15ef876aa48d75a73a355" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.6.2" }, "parso": { @@ -2642,7 +2650,7 @@ "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.9.0.post0" }, "pyyaml": { @@ -2772,7 +2780,7 @@ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, "smmap": { @@ -2803,7 +2811,7 @@ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.10.2" }, "tomli": { @@ -2835,7 +2843,7 @@ "sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3", "sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429" ], - "markers": "python_version >= '3.6'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", "version": "==1.26.19" }, "watchdog": { diff --git a/api/conf/celery.py b/api/conf/celery.py index 7971316e80..40c118bc24 100644 --- a/api/conf/celery.py +++ b/api/conf/celery.py @@ -3,7 +3,8 @@ from celery import Celery from celery.schedules import crontab from dbt_copilot_python.celery_health_check import healthcheck -from dbt_copilot_python.utility import is_copilot + +from django.conf import settings # Set the default Django settings module for the 'celery' program. os.environ.setdefault("DJANGO_SETTINGS_MODULE", "api.conf.settings") @@ -43,5 +44,5 @@ }, } -if is_copilot: +if settings.IS_ENV_DBT_PLATFORM: celery_app = healthcheck.setup(app) diff --git a/api/conf/settings.py b/api/conf/settings.py index 18852bb95a..f72bcdb71a 100644 --- a/api/conf/settings.py +++ b/api/conf/settings.py @@ -9,8 +9,11 @@ from sentry_sdk.integrations.django import DjangoIntegration from dbt_copilot_python.network import setup_allowed_hosts +from dbt_copilot_python.database import database_url_from_env from dbt_copilot_python.utility import is_copilot +import dj_database_url + from django_log_formatter_ecs import ECSFormatter from django_log_formatter_asim import ASIMFormatter @@ -50,6 +53,12 @@ DEBUG = env("DEBUG") +ENV = env("ENV") +VCAP_SERVICES = env.json("VCAP_SERVICES", {}) + +IS_ENV_DBT_PLATFORM = is_copilot() +IS_ENV_GOV_PAAS = bool(VCAP_SERVICES) + # Please use this to Enable/Disable the Admin site ADMIN_ENABLED = env("ADMIN_ENABLED", default=False) @@ -245,55 +254,13 @@ LETTER_TEMPLATES_DIRECTORY = os.path.join(BASE_DIR, "letter_templates", "templates", "letter_templates") -# Database -DATABASES = {"default": env.db()} # https://docs.djangoproject.com/en/2.1/ref/settings/#databases DEFAULT_AUTO_FIELD = "django.db.models.AutoField" # AWS -VCAP_SERVICES = env.json("VCAP_SERVICES", {}) S3_BUCKET_TAG_FILE_UPLOADS = "file-uploads" -if VCAP_SERVICES: - if "aws-s3-bucket" not in VCAP_SERVICES: - raise Exception("S3 Bucket not bound to environment") - - for bucket_details in VCAP_SERVICES["aws-s3-bucket"]: - if S3_BUCKET_TAG_FILE_UPLOADS in bucket_details["tags"]: - aws_credentials = bucket_details["credentials"] - AWS_ENDPOINT_URL = None - AWS_ACCESS_KEY_ID = aws_credentials["aws_access_key_id"] - AWS_SECRET_ACCESS_KEY = aws_credentials["aws_secret_access_key"] - AWS_REGION = aws_credentials["aws_region"] - AWS_STORAGE_BUCKET_NAME = aws_credentials["bucket_name"] -else: - AWS_ENDPOINT_URL = env("AWS_ENDPOINT_URL", default=None) - AWS_ACCESS_KEY_ID = env("AWS_ACCESS_KEY_ID") - AWS_SECRET_ACCESS_KEY = env("AWS_SECRET_ACCESS_KEY") - AWS_REGION = env("AWS_REGION") - AWS_STORAGE_BUCKET_NAME = env("AWS_STORAGE_BUCKET_NAME") - -if "redis" in VCAP_SERVICES: - REDIS_BASE_URL = VCAP_SERVICES["redis"][0]["credentials"]["uri"] -else: - REDIS_BASE_URL = env("REDIS_BASE_URL", default=None) - - -def _build_redis_url(base_url, db_number, **query_args): - encoded_query_args = urlencode(query_args) - return f"{base_url}/{db_number}?{encoded_query_args}" - - -if REDIS_BASE_URL: - # Give celery tasks their own redis DB - future uses of redis should use a different DB - REDIS_CELERY_DB = env("REDIS_CELERY_DB", default=0) - is_redis_ssl = REDIS_BASE_URL.startswith("rediss://") - url_args = {"ssl_cert_reqs": "CERT_REQUIRED"} if is_redis_ssl else {} - - CELERY_BROKER_URL = _build_redis_url(REDIS_BASE_URL, REDIS_CELERY_DB, **url_args) - CELERY_RESULT_BACKEND = CELERY_BROKER_URL - CELERY_ALWAYS_EAGER = env.bool("CELERY_ALWAYS_EAGER", False) CELERY_TASK_ALWAYS_EAGER = env.bool("CELERY_TASK_ALWAYS_EAGER", False) CELERY_TASK_STORE_EAGER_RESULT = env.bool("CELERY_TASK_STORE_EAGER_RESULT", False) @@ -342,61 +309,9 @@ def _build_redis_url(base_url, db_number, **query_args): ELASTICSEARCH_PRODUCT_INDEX_ALIAS = env.str("ELASTICSEARCH_PRODUCT_INDEX_ALIAS", "products-alias") ELASTICSEARCH_APPLICATION_INDEX_ALIAS = env.str("ELASTICSEARCH_APPLICATION_INDEX_ALIAS", "application-alias") -# Elasticsearch configuration -LITE_API_ENABLE_ES = env.bool("LITE_API_ENABLE_ES", False) -if LITE_API_ENABLE_ES: - ELASTICSEARCH_DSL = { - "default": {"hosts": env.str("ELASTICSEARCH_HOST")}, - } - - ENABLE_SPIRE_SEARCH = env.bool("ENABLE_SPIRE_SEARCH", False) - - ELASTICSEARCH_PRODUCT_INDEXES = {"LITE": ELASTICSEARCH_PRODUCT_INDEX_ALIAS} - ELASTICSEARCH_APPLICATION_INDEXES = {"LITE": ELASTICSEARCH_APPLICATION_INDEX_ALIAS} - SPIRE_APPLICATION_INDEX_NAME = env.str("SPIRE_APPLICATION_INDEX_NAME", "spire-application-alias") - SPIRE_PRODUCT_INDEX_NAME = env.str("SPIRE_PRODUCT_INDEX_NAME", "spire-products-alias") - - if ENABLE_SPIRE_SEARCH: - ELASTICSEARCH_APPLICATION_INDEXES["SPIRE"] = SPIRE_APPLICATION_INDEX_NAME - ELASTICSEARCH_PRODUCT_INDEXES["SPIRE"] = SPIRE_PRODUCT_INDEX_NAME - - INSTALLED_APPS += [ - "django_elasticsearch_dsl", - "django_elasticsearch_dsl_drf", - ] - -ENV = env("ENV") DENIAL_REASONS_DELETION_LOGGER = "denial_reasons_deletion_logger" -LOGGING = { - "version": 1, - "disable_existing_loggers": False, - "handlers": { - "sentry": {"class": "sentry_sdk.integrations.logging.EventHandler"}, - }, - "loggers": { - DENIAL_REASONS_DELETION_LOGGER: {"handlers": ["sentry"], "level": logging.WARNING}, - }, -} - -if ENV == "local": - LOGGING.update({"formatters": {"simple": {"format": "{asctime} {levelname} {message}", "style": "{"}}}) - LOGGING["handlers"].update({"stdout": {"class": "logging.StreamHandler", "formatter": "simple"}}) - LOGGING.update({"root": {"handlers": ["stdout"], "level": env("LOG_LEVEL").upper()}}) - - -elif not is_copilot(): - LOGGING.update({"formatters": {"ecs_formatter": {"()": ECSFormatter}}}) - LOGGING["handlers"].update({"ecs": {"class": "logging.StreamHandler", "formatter": "ecs_formatter"}}) - LOGGING.update({"root": {"handlers": ["ecs"], "level": env("LOG_LEVEL").upper()}}) - -else: - LOGGING.update({"formatters": {"asim_formatter": {"()": ASIMFormatter}}}) - LOGGING["handlers"].update({"asim": {"class": "logging.StreamHandler", "formatter": "asim_formatter"}}) - LOGGING.update({"root": {"handlers": ["asim"], "level": env("LOG_LEVEL").upper()}}) - - # Sentry if env.str("SENTRY_DSN", ""): sentry_sdk.init( @@ -520,11 +435,117 @@ def _build_redis_url(base_url, db_number, **query_args): S3_BUCKET_TAG_ANONYMISER_DESTINATION = "anonymiser" -if VCAP_SERVICES: +ELASTICSEARCH_SANCTION_INDEX_ALIAS = env.str("ELASTICSEARCH_SANCTION_INDEX_ALIAS", "sanctions-alias") +ELASTICSEARCH_DENIALS_INDEX_ALIAS = env.str("ELASTICSEARCH_DENIALS_INDEX_ALIAS", "denials-alias") +ELASTICSEARCH_PRODUCT_INDEX_ALIAS = env.str("ELASTICSEARCH_PRODUCT_INDEX_ALIAS", "products-alias") +ELASTICSEARCH_APPLICATION_INDEX_ALIAS = env.str("ELASTICSEARCH_APPLICATION_INDEX_ALIAS", "application-alias") + +DB_ANONYMISER_CONFIG_LOCATION = Path(BASE_DIR) / "conf" / "anonymise_model_config.yaml" +DB_ANONYMISER_DUMP_FILE_NAME = env.str("DB_ANONYMISER_DUMP_FILE_NAME", "anonymised.sql") + + +def _build_redis_url(base_url, db_number, **query_args): + encoded_query_args = urlencode(query_args) + return f"{base_url}/{db_number}?{encoded_query_args}" + + +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "handlers": { + "sentry": {"class": "sentry_sdk.integrations.logging.EventHandler"}, + }, + "loggers": { + DENIAL_REASONS_DELETION_LOGGER: {"handlers": ["sentry"], "level": logging.WARNING}, + }, +} + +if IS_ENV_DBT_PLATFORM: + ALLOWED_HOSTS = setup_allowed_hosts(ALLOWED_HOSTS) + + DATABASES = {"default": dj_database_url.config(default=database_url_from_env("DATABASE_CREDENTIALS"))} + CELERY_BROKER_URL = env("CELERY_BROKER_URL", default=None) + CELERY_RESULT_BACKEND = CELERY_BROKER_URL + REDIS_BASE_URL = env("REDIS_BASE_URL", default=None) + + # Elasticsearch configuration + LITE_API_ENABLE_ES = env.bool("LITE_API_ENABLE_ES", False) + if LITE_API_ENABLE_ES: + ELASTICSEARCH_DSL = { + "default": {"hosts": env.str("OPENSEARCH_ENDPOINT")}, + } + + ENABLE_SPIRE_SEARCH = env.bool("ENABLE_SPIRE_SEARCH", False) + + ELASTICSEARCH_PRODUCT_INDEXES = {"LITE": ELASTICSEARCH_PRODUCT_INDEX_ALIAS} + ELASTICSEARCH_APPLICATION_INDEXES = {"LITE": ELASTICSEARCH_APPLICATION_INDEX_ALIAS} + SPIRE_APPLICATION_INDEX_NAME = env.str("SPIRE_APPLICATION_INDEX_NAME", "spire-application-alias") + SPIRE_PRODUCT_INDEX_NAME = env.str("SPIRE_PRODUCT_INDEX_NAME", "spire-products-alias") + + if ENABLE_SPIRE_SEARCH: + ELASTICSEARCH_APPLICATION_INDEXES["SPIRE"] = SPIRE_APPLICATION_INDEX_NAME + ELASTICSEARCH_PRODUCT_INDEXES["SPIRE"] = SPIRE_PRODUCT_INDEX_NAME + + INSTALLED_APPS += [ + "django_elasticsearch_dsl", + "django_elasticsearch_dsl_drf", + ] + + LOGGING.update({"formatters": {"asim_formatter": {"()": ASIMFormatter}}}) + LOGGING["handlers"].update({"asim": {"class": "logging.StreamHandler", "formatter": "asim_formatter"}}) + LOGGING.update({"root": {"handlers": ["asim"], "level": env("LOG_LEVEL").upper()}}) + +elif IS_ENV_GOV_PAAS: + # This has repeating code as this section can be deleted once migrated to DBT Platform + # Database + DATABASES = {"default": env.db()} # https://docs.djangoproject.com/en/2.1/ref/settings/#databases + # redis + REDIS_BASE_URL = VCAP_SERVICES["redis"][0]["credentials"]["uri"] + + if REDIS_BASE_URL: + # Give celery tasks their own redis DB - future uses of redis should use a different DB + REDIS_CELERY_DB = env("REDIS_CELERY_DB", default=0) + is_redis_ssl = REDIS_BASE_URL.startswith("rediss://") + url_args = {"ssl_cert_reqs": "CERT_REQUIRED"} if is_redis_ssl else {} + + CELERY_BROKER_URL = _build_redis_url(REDIS_BASE_URL, REDIS_CELERY_DB, **url_args) + CELERY_RESULT_BACKEND = CELERY_BROKER_URL + + # Elasticsearch configuration + LITE_API_ENABLE_ES = env.bool("LITE_API_ENABLE_ES", False) + if LITE_API_ENABLE_ES: + ELASTICSEARCH_DSL = { + "default": {"hosts": env.str("ELASTICSEARCH_HOST")}, + } + + ENABLE_SPIRE_SEARCH = env.bool("ENABLE_SPIRE_SEARCH", False) + + ELASTICSEARCH_PRODUCT_INDEXES = {"LITE": ELASTICSEARCH_PRODUCT_INDEX_ALIAS} + ELASTICSEARCH_APPLICATION_INDEXES = {"LITE": ELASTICSEARCH_APPLICATION_INDEX_ALIAS} + SPIRE_APPLICATION_INDEX_NAME = env.str("SPIRE_APPLICATION_INDEX_NAME", "spire-application-alias") + SPIRE_PRODUCT_INDEX_NAME = env.str("SPIRE_PRODUCT_INDEX_NAME", "spire-products-alias") + + if ENABLE_SPIRE_SEARCH: + ELASTICSEARCH_APPLICATION_INDEXES["SPIRE"] = SPIRE_APPLICATION_INDEX_NAME + ELASTICSEARCH_PRODUCT_INDEXES["SPIRE"] = SPIRE_PRODUCT_INDEX_NAME + + INSTALLED_APPS += [ + "django_elasticsearch_dsl", + "django_elasticsearch_dsl_drf", + ] + # AWS if "aws-s3-bucket" not in VCAP_SERVICES: raise Exception("S3 Bucket not bound to environment") for bucket_details in VCAP_SERVICES["aws-s3-bucket"]: + if S3_BUCKET_TAG_FILE_UPLOADS in bucket_details["tags"]: + aws_credentials = bucket_details["credentials"] + AWS_ENDPOINT_URL = None + AWS_ACCESS_KEY_ID = aws_credentials["aws_access_key_id"] + AWS_SECRET_ACCESS_KEY = aws_credentials["aws_secret_access_key"] + AWS_REGION = aws_credentials["aws_region"] + AWS_STORAGE_BUCKET_NAME = aws_credentials["bucket_name"] + if S3_BUCKET_TAG_ANONYMISER_DESTINATION in bucket_details["tags"]: aws_credentials = bucket_details["credentials"] DB_ANONYMISER_AWS_ENDPOINT_URL = None @@ -532,17 +553,62 @@ def _build_redis_url(base_url, db_number, **query_args): DB_ANONYMISER_AWS_SECRET_ACCESS_KEY = aws_credentials["aws_secret_access_key"] DB_ANONYMISER_AWS_REGION = aws_credentials["aws_region"] DB_ANONYMISER_AWS_STORAGE_BUCKET_NAME = aws_credentials["bucket_name"] + + LOGGING.update({"formatters": {"ecs_formatter": {"()": ECSFormatter}}}) + LOGGING["handlers"].update({"ecs": {"class": "logging.StreamHandler", "formatter": "ecs_formatter"}}) + LOGGING.update({"root": {"handlers": ["ecs"], "level": env("LOG_LEVEL").upper()}}) + else: + # Local configurations and CircleCI + # Database + DATABASES = {"default": env.db()} # https://docs.djangoproject.com/en/2.1/ref/settings/#databases + # redis + REDIS_BASE_URL = env("REDIS_BASE_URL", default=None) + + if REDIS_BASE_URL: + # Give celery tasks their own redis DB - future uses of redis should use a different DB + REDIS_CELERY_DB = env("REDIS_CELERY_DB", default=0) + is_redis_ssl = REDIS_BASE_URL.startswith("rediss://") + url_args = {"ssl_cert_reqs": "CERT_REQUIRED"} if is_redis_ssl else {} + + CELERY_BROKER_URL = _build_redis_url(REDIS_BASE_URL, REDIS_CELERY_DB, **url_args) + CELERY_RESULT_BACKEND = CELERY_BROKER_URL + + # Elasticsearch configuration + LITE_API_ENABLE_ES = env.bool("LITE_API_ENABLE_ES", False) + if LITE_API_ENABLE_ES: + ELASTICSEARCH_DSL = { + "default": {"hosts": env.str("ELASTICSEARCH_HOST")}, + } + + ENABLE_SPIRE_SEARCH = env.bool("ENABLE_SPIRE_SEARCH", False) + + ELASTICSEARCH_PRODUCT_INDEXES = {"LITE": ELASTICSEARCH_PRODUCT_INDEX_ALIAS} + ELASTICSEARCH_APPLICATION_INDEXES = {"LITE": ELASTICSEARCH_APPLICATION_INDEX_ALIAS} + SPIRE_APPLICATION_INDEX_NAME = env.str("SPIRE_APPLICATION_INDEX_NAME", "spire-application-alias") + SPIRE_PRODUCT_INDEX_NAME = env.str("SPIRE_PRODUCT_INDEX_NAME", "spire-products-alias") + + if ENABLE_SPIRE_SEARCH: + ELASTICSEARCH_APPLICATION_INDEXES["SPIRE"] = SPIRE_APPLICATION_INDEX_NAME + ELASTICSEARCH_PRODUCT_INDEXES["SPIRE"] = SPIRE_PRODUCT_INDEX_NAME + + INSTALLED_APPS += [ + "django_elasticsearch_dsl", + "django_elasticsearch_dsl_drf", + ] + # AWS + AWS_ENDPOINT_URL = env("AWS_ENDPOINT_URL", default=None) + AWS_ACCESS_KEY_ID = env("AWS_ACCESS_KEY_ID") + AWS_SECRET_ACCESS_KEY = env("AWS_SECRET_ACCESS_KEY") + AWS_REGION = env("AWS_REGION") + AWS_STORAGE_BUCKET_NAME = env("AWS_STORAGE_BUCKET_NAME") + DB_ANONYMISER_AWS_ENDPOINT_URL = AWS_ENDPOINT_URL DB_ANONYMISER_AWS_ACCESS_KEY_ID = env("DB_ANONYMISER_AWS_ACCESS_KEY_ID", default=None) DB_ANONYMISER_AWS_SECRET_ACCESS_KEY = env("DB_ANONYMISER_AWS_SECRET_ACCESS_KEY", default=None) DB_ANONYMISER_AWS_REGION = env("DB_ANONYMISER_AWS_REGION", default=None) DB_ANONYMISER_AWS_STORAGE_BUCKET_NAME = env("DB_ANONYMISER_AWS_STORAGE_BUCKET_NAME", default=None) -DB_ANONYMISER_CONFIG_LOCATION = Path(BASE_DIR) / "conf" / "anonymise_model_config.yaml" -DB_ANONYMISER_DUMP_FILE_NAME = env.str("DB_ANONYMISER_DUMP_FILE_NAME", "anonymised.sql") - - -# DBT Platform Spectic config -if is_copilot: - ALLOWED_HOSTS = setup_allowed_hosts(ALLOWED_HOSTS) + LOGGING.update({"formatters": {"simple": {"format": "{asctime} {levelname} {message}", "style": "{"}}}) + LOGGING["handlers"].update({"stdout": {"class": "logging.StreamHandler", "formatter": "simple"}}) + LOGGING.update({"root": {"handlers": ["stdout"], "level": env("LOG_LEVEL").upper()}}) From b72d20238d8d69d825e3ed0e830803bf3fdca109 Mon Sep 17 00:00:00 2001 From: Henry Cooksley Date: Thu, 27 Jun 2024 14:39:31 +0100 Subject: [PATCH 45/73] Update ApplicationPartyView put method to check case status --- api/applications/views/parties.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/api/applications/views/parties.py b/api/applications/views/parties.py index f16d8b400b..7032325c6e 100644 --- a/api/applications/views/parties.py +++ b/api/applications/views/parties.py @@ -19,6 +19,7 @@ from api.parties.models import Party from api.parties.serializers import PartySerializer from api.users.models import ExporterUser +from api.staticdata.statuses.enums import CaseStatusEnum class ApplicationPartyView(APIView): @@ -136,6 +137,14 @@ def get(self, request, **kwargs): @authorised_to_view_application(ExporterUser) def put(self, request, **kwargs): + # Check if the case is in draft or applicant_editing status + case_status = self.application.get_case().status.status + if case_status not in [CaseStatusEnum.APPLICANT_EDITING, CaseStatusEnum.DRAFT]: + return JsonResponse( + data={"errors": [f"The case cannot be edited in {case_status} status"]}, + status=status.HTTP_403_FORBIDDEN, + ) + serializer = PartySerializer(instance=self.party, data=request.data, partial=True) if not serializer.is_valid(): return JsonResponse(data={"errors": serializer.errors}, status=status.HTTP_400_BAD_REQUEST) From 03459d56a8dd58cb296bbbb4452c189aecbab76a Mon Sep 17 00:00:00 2001 From: Arun Siluvery Date: Thu, 27 Jun 2024 14:13:26 +0100 Subject: [PATCH 46/73] Good cannot be edited if it is already submitted on an application --- api/assessments/tests/test_views.py | 5 +++++ api/goods/tests/test_submitting_goods.py | 15 ++++++++++++++- api/goods/views.py | 11 ++++++----- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/api/assessments/tests/test_views.py b/api/assessments/tests/test_views.py index 48f4eaab14..913cf5bb17 100644 --- a/api/assessments/tests/test_views.py +++ b/api/assessments/tests/test_views.py @@ -216,6 +216,11 @@ def test_valid_data_updates_single_record(self): "report_summary": good_on_application.report_summary, } + # ensure verified good cannot be edited + url = reverse("goods:good", kwargs={"pk": good.id}) + response = self.client.put(url, {"is_archived": True}, **self.exporter_headers) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + def test_making_a_good_uncontrolled_clears_report_fields(self): # Setting is_good_controlled to False should set report_summary_prefix and report_summary_subject to None good_on_application = self.good_on_application diff --git a/api/goods/tests/test_submitting_goods.py b/api/goods/tests/test_submitting_goods.py index 9bb2fe0764..8e0c57a22f 100644 --- a/api/goods/tests/test_submitting_goods.py +++ b/api/goods/tests/test_submitting_goods.py @@ -138,7 +138,7 @@ def test_submitted_good_cannot_be_edited(self): good = Good.objects.get() url = reverse("goods:good", kwargs={"pk": good.id}) - response = self.client.put(url, {}, **self.exporter_headers) + response = self.client.put(url, {"name": "Updated name"}, **self.exporter_headers) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) def test_unsubmitted_good_can_be_edited(self): @@ -194,3 +194,16 @@ def test_deleted_good_removed_from_all_drafts_they_existed_in(self): self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(Good.objects.all().count(), 0) self.assertEqual(GoodOnApplication.objects.count(), 0) + + def test_submitted_good_cannot_be_edited_good_details_url(self): + """ + Tests that the good cannot be edited after submission + """ + draft = self.create_draft_standard_application(self.organisation) + self.submit_application(application=draft) + + good = Good.objects.get() + url = reverse("goods:good_details", kwargs={"pk": good.id}) + + response = self.client.put(url, {"name": "Updated name"}, **self.exporter_headers) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) diff --git a/api/goods/views.py b/api/goods/views.py index 90bdeff985..2a9faf0295 100644 --- a/api/goods/views.py +++ b/api/goods/views.py @@ -247,6 +247,9 @@ def put(self, request, pk): good = get_good(pk) data = request.data.copy() + if good.status != GoodStatus.DRAFT: + raise BadRequestError({"non_field_errors": [strings.Goods.CANNOT_EDIT_GOOD]}) + # return bad request if trying to edit software_or_technology details outside of category group 3 if good.item_category in ItemCategory.group_one and "software_or_technology_details" in data: raise BadRequestError({"non_field_errors": [strings.Goods.CANNOT_SET_DETAILS_ERROR]}) @@ -259,9 +262,6 @@ def put(self, request, pk): if good.item_category not in ItemCategory.group_two and data.get("firearm_details"): check_if_firearm_details_edited_on_unsupported_good(data) - if good.status == GoodStatus.SUBMITTED: - raise BadRequestError({"non_field_errors": [strings.Goods.CANNOT_EDIT_GOOD]}) - # check if the user is registered firearm dealer if good.item_category == ItemCategory.GROUP2_FIREARMS and good.firearm_details.type in FIREARMS_CORE_TYPES: is_rfd = has_valid_certificate( @@ -320,9 +320,10 @@ def put(self, request, pk): if good.organisation_id != get_request_user_organisation_id(request): raise PermissionDenied() - if good.status == GoodStatus.SUBMITTED: + if good.status != GoodStatus.DRAFT: return JsonResponse( - data={"errors": "This good is already on a submitted application"}, status=status.HTTP_400_BAD_REQUEST + data={"errors": f"Good {good.name} is already used on a submitted application and cannot be edited"}, + status=status.HTTP_400_BAD_REQUEST, ) data = request.data.copy() From f3a8299336d261eb5628449b3d43a5aa9fae128e Mon Sep 17 00:00:00 2001 From: Kevin Carrogan Date: Thu, 27 Jun 2024 17:24:21 +0100 Subject: [PATCH 47/73] Default to checking out master frontend for e2e tests --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 32a11e4246..9bebaca298 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -510,6 +510,7 @@ jobs: TEMPORARY_BRANCH_NAME+=$(cat /proc/sys/kernel/random/uuid) git clone git@github.com:uktrade/lite-frontend.git cd lite-frontend + git checkout master git config user.email $GIT_EMAIL git config user.name "LITE CI" git checkout -b $TEMPORARY_BRANCH_NAME From 5384392db2682adf22dc1d9117908e35ee53e8c2 Mon Sep 17 00:00:00 2001 From: Henry Cooksley Date: Thu, 27 Jun 2024 17:44:56 +0100 Subject: [PATCH 48/73] Add TestApplicationPartyView tests --- api/applications/views/parties.py | 2 +- api/applications/views/tests/test_parties.py | 55 ++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 api/applications/views/tests/test_parties.py diff --git a/api/applications/views/parties.py b/api/applications/views/parties.py index 7032325c6e..1bfaec4afe 100644 --- a/api/applications/views/parties.py +++ b/api/applications/views/parties.py @@ -141,7 +141,7 @@ def put(self, request, **kwargs): case_status = self.application.get_case().status.status if case_status not in [CaseStatusEnum.APPLICANT_EDITING, CaseStatusEnum.DRAFT]: return JsonResponse( - data={"errors": [f"The case cannot be edited in {case_status} status"]}, + data={"errors": [f"The {self.party.type} party cannot be edited in {case_status} status"]}, status=status.HTTP_403_FORBIDDEN, ) diff --git a/api/applications/views/tests/test_parties.py b/api/applications/views/tests/test_parties.py new file mode 100644 index 0000000000..e6b98f36b8 --- /dev/null +++ b/api/applications/views/tests/test_parties.py @@ -0,0 +1,55 @@ +from django.urls import reverse +from parameterized import parameterized +from rest_framework import status + +from api.applications.tests.factories import ( + StandardApplicationFactory, + PartyOnApplicationFactory, +) +from api.staticdata.statuses.models import CaseStatus +from test_helpers.clients import DataTestClient + + +class TestApplicationPartyView(DataTestClient): + + def setUp(self): + super().setUp() + self.application = StandardApplicationFactory(organisation=self.organisation) + self.case_statuses = [ + case_status.status + for case_status in CaseStatus.objects.all() + if case_status.status not in ["draft", "applicant_editing"] + ] + self.data = {"name": "End user", "address": "1 Example Street", "country": {"id": "FR", "name": "France"}} + self.party_on_application = PartyOnApplicationFactory(application=self.application) + self.url = reverse( + "applications:party", + kwargs={"pk": str(self.application.pk), "party_pk": str(self.party_on_application.party.pk)}, + ) + + def test_draft_application_can_update_party_detail(self): + self.application.status = CaseStatus.objects.get(status="draft") + self.application.save() + response = self.client.put(self.url, **self.exporter_headers, data=self.data) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + def test_editing_application_can_update_party_detail(self): + self.application.status = CaseStatus.objects.get(status="applicant_editing") + self.application.save() + response = self.client.put(self.url, **self.exporter_headers, data=self.data) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + def test_other_status_cannot_update_party_detail(self): + for case_status in self.case_statuses: + self.application.status = CaseStatus.objects.get(status=case_status) + self.application.save() + response = self.client.put(self.url, **self.exporter_headers, data=self.data) + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + self.assertEqual( + response.json(), + { + "errors": [ + f"The {self.party_on_application.party.type} party cannot be edited in {case_status} status" + ] + }, + ) From c6f9a51da088a49cefeb6b55c0fb35c26abf5bc3 Mon Sep 17 00:00:00 2001 From: Kevin Carrogan Date: Thu, 27 Jun 2024 17:24:21 +0100 Subject: [PATCH 49/73] Default to checking out master frontend for e2e tests --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 32a11e4246..9bebaca298 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -510,6 +510,7 @@ jobs: TEMPORARY_BRANCH_NAME+=$(cat /proc/sys/kernel/random/uuid) git clone git@github.com:uktrade/lite-frontend.git cd lite-frontend + git checkout master git config user.email $GIT_EMAIL git config user.name "LITE CI" git checkout -b $TEMPORARY_BRANCH_NAME From 50d099c4c62d506319e2b2bf232f6d61627b6f56 Mon Sep 17 00:00:00 2001 From: Kevin Carrogan Date: Mon, 1 Jul 2024 12:03:17 +0100 Subject: [PATCH 50/73] Stops audit entries that are related to a GoodOnApplication from being removed when a GoodOnApplication object is deleted --- api/applications/libraries/get_applications.py | 1 - api/applications/models.py | 9 +-------- api/applications/serializers/good.py | 10 +++++++++- api/applications/tests/test_audit_trail.py | 15 +++++++++++++++ 4 files changed, 25 insertions(+), 10 deletions(-) create mode 100644 api/applications/tests/test_audit_trail.py diff --git a/api/applications/libraries/get_applications.py b/api/applications/libraries/get_applications.py index bb6948c87a..2e7ea726af 100644 --- a/api/applications/libraries/get_applications.py +++ b/api/applications/libraries/get_applications.py @@ -47,7 +47,6 @@ def get_application(pk, organisation_id=None): "goods__regime_entries__subsection__regime", "goods__good__report_summary_prefix", "goods__good__report_summary_subject", - "goods__audit_trail", "goods__goodonapplicationdocument_set", "goods__goodonapplicationdocument_set__user", "goods__good_on_application_internal_documents", diff --git a/api/applications/models.py b/api/applications/models.py index ac93c768e9..6c0fb11430 100644 --- a/api/applications/models.py +++ b/api/applications/models.py @@ -1,6 +1,5 @@ import uuid -from django.contrib.contenttypes.fields import GenericRelation from django.contrib.postgres.fields import ArrayField from django.db import models, transaction from django.utils import timezone @@ -16,7 +15,7 @@ ) from api.appeals.models import Appeal from api.applications.managers import BaseApplicationManager -from api.audit_trail.models import Audit, AuditType +from api.audit_trail.models import AuditType from api.audit_trail import service as audit_trail_service from api.cases.enums import CaseTypeEnum from api.cases.models import Case, CaseQueue @@ -512,12 +511,6 @@ class GoodOnApplication(AbstractGoodOnApplication, Clonable): # Exhibition applications are the only applications that contain the following as such may be null item_type = models.CharField(choices=ItemType.choices, max_length=10, null=True, blank=True, default=None) other_item_type = models.CharField(max_length=100, null=True, blank=True, default=None) - audit_trail = GenericRelation( - Audit, - related_query_name="good_on_application", - content_type_field="action_object_content_type", - object_id_field="action_object_object_id", - ) control_list_entries = models.ManyToManyField(ControlListEntry, through=GoodOnApplicationControlListEntry) regime_entries = models.ManyToManyField(RegimeEntry, through=GoodOnApplicationRegimeEntry) diff --git a/api/applications/serializers/good.py b/api/applications/serializers/good.py index dfe8f88a0b..8f673b46e9 100644 --- a/api/applications/serializers/good.py +++ b/api/applications/serializers/good.py @@ -2,6 +2,7 @@ from rest_framework.fields import DecimalField, ChoiceField, BooleanField from rest_framework.relations import PrimaryKeyRelatedField +from django.contrib.contenttypes.models import ContentType from django.forms.models import model_to_dict from api.applications.models import ( @@ -15,6 +16,7 @@ ) from api.audit_trail import service as audit_trail_service from api.audit_trail.enums import AuditType +from api.audit_trail.models import Audit from api.audit_trail.serializers import AuditSerializer from api.applications.enums import NSGListType from api.cases.enums import CaseTypeEnum @@ -153,7 +155,13 @@ def get_audit_trail(self, instance): # this serializer is used by a few views. Most views do not need to know audit trail if not self.context.get("include_audit_trail"): return [] - return AuditSerializer(instance.audit_trail.all(), many=True).data + action_object_content_type = ContentType.objects.get_for_model(GoodOnApplication) + action_object_object_id = instance.pk + audits = Audit.objects.filter( + action_object_content_type=action_object_content_type, + action_object_object_id=action_object_object_id, + ) + return AuditSerializer(audits, many=True).data def update(self, instance, validated_data): if "firearm_details" in validated_data: diff --git a/api/applications/tests/test_audit_trail.py b/api/applications/tests/test_audit_trail.py new file mode 100644 index 0000000000..3dbdc16076 --- /dev/null +++ b/api/applications/tests/test_audit_trail.py @@ -0,0 +1,15 @@ +from api.audit_trail.tests.factories import AuditFactory +from test_helpers.clients import DataTestClient + + +class GoodOnApplicationAuditTrailTests(DataTestClient): + def test_removing_object_keeps_audit_trail(self): + application = self.create_draft_standard_application(self.organisation) + good_on_application = application.goods.first() + audit = AuditFactory( + action_object=good_on_application, + ) + + good_on_application.delete() + audit.refresh_from_db() + self.assertIsNone(audit.action_object) From 3373345468126023f9181cc6c7c1cd040d62cd67 Mon Sep 17 00:00:00 2001 From: Tomos Williams Date: Mon, 1 Jul 2024 15:25:16 +0100 Subject: [PATCH 51/73] got a little stuck reindexing, figured it was worth adding to the readme/makefile --- README.md | 2 ++ makefile | 3 +++ 2 files changed, 5 insertions(+) diff --git a/README.md b/README.md index 32849e102e..d4492050d9 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,8 @@ Service for handling backend calls in LITE. - Starting the service - In general you can use `docker-compose up --build` if you want to make sure new changes are included in the build +- Indexing elasticsearch + - If this is something you require, you can run `make rebuild-elastic` to rebuild the elastic indexes using your local db. ### Known issues when running with Docker diff --git a/makefile b/makefile index 2748baa254..5576fb5db7 100644 --- a/makefile +++ b/makefile @@ -67,3 +67,6 @@ stop-e2e: test: ./manage.py test + +rebuild-elastic: + docker exec -it api pipenv run ./manage.py search_index --rebuild From 7b71d4c504bf690b83ea8b16d2b2bb766f205be8 Mon Sep 17 00:00:00 2001 From: Kevin Carrogan Date: Mon, 1 Jul 2024 16:20:20 +0100 Subject: [PATCH 52/73] fixup --- api/applications/tests/test_audit_trail.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/api/applications/tests/test_audit_trail.py b/api/applications/tests/test_audit_trail.py index 3dbdc16076..0565fb44a0 100644 --- a/api/applications/tests/test_audit_trail.py +++ b/api/applications/tests/test_audit_trail.py @@ -1,15 +1,22 @@ +from api.audit_trail.models import Audit from api.audit_trail.tests.factories import AuditFactory + from test_helpers.clients import DataTestClient class GoodOnApplicationAuditTrailTests(DataTestClient): def test_removing_object_keeps_audit_trail(self): application = self.create_draft_standard_application(self.organisation) + Audit.objects.all().delete() + good_on_application = application.goods.first() audit = AuditFactory( action_object=good_on_application, ) + self.assertEqual(Audit.objects.count(), 1) good_on_application.delete() - audit.refresh_from_db() + + self.assertEqual(Audit.objects.count(), 1) + audit = Audit.objects.get(pk=audit.pk) self.assertIsNone(audit.action_object) From 4973efa55135e05a4c08892757277c492b6e31cf Mon Sep 17 00:00:00 2001 From: Tomos Williams Date: Tue, 2 Jul 2024 14:23:50 +0100 Subject: [PATCH 53/73] elasticsearch -> search --- README.md | 4 ++-- makefile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d4492050d9..72a4209f7d 100644 --- a/README.md +++ b/README.md @@ -71,8 +71,8 @@ Service for handling backend calls in LITE. - Starting the service - In general you can use `docker-compose up --build` if you want to make sure new changes are included in the build -- Indexing elasticsearch - - If this is something you require, you can run `make rebuild-elastic` to rebuild the elastic indexes using your local db. +- Indexing search + - If this is something you require, you can run `make rebuild-search` to rebuild the search indexes using your local db. ### Known issues when running with Docker diff --git a/makefile b/makefile index 5576fb5db7..0bb2c9717c 100644 --- a/makefile +++ b/makefile @@ -68,5 +68,5 @@ stop-e2e: test: ./manage.py test -rebuild-elastic: - docker exec -it api pipenv run ./manage.py search_index --rebuild +rebuild-search: + docker exec -it api pipenv run ./manage.py search_index -f --rebuild From 24710e61e89a5149904da10d228f5f13f34aa768 Mon Sep 17 00:00:00 2001 From: Gurdeep Atwal Date: Fri, 28 Jun 2024 15:27:48 +0100 Subject: [PATCH 54/73] fix forward slash searching --- .../tests/test_matching_denials.py | 11 ++-- api/core/search/filter_backends.py | 14 ++++ api/core/search/validators.py | 3 + api/external_data/tests/denial_valid.csv | 2 + api/external_data/tests/test_views.py | 66 +++++++++++++++++-- api/search/product/tests/test_helpers.py | 2 +- api/search/product/tests/test_views.py | 3 +- 7 files changed, 89 insertions(+), 12 deletions(-) diff --git a/api/applications/tests/test_matching_denials.py b/api/applications/tests/test_matching_denials.py index 0a59a460ca..940f9f38f3 100644 --- a/api/applications/tests/test_matching_denials.py +++ b/api/applications/tests/test_matching_denials.py @@ -17,9 +17,12 @@ def setUp(self): file_path = os.path.join(settings.BASE_DIR, "external_data/tests/denial_valid.csv") with open(file_path, "rb") as f: content = f.read() + f.seek(0) + self.total_denials = len(f.readlines()) - 1 + response = self.client.post(reverse("external_data:denial-list"), {"csv_file": content}, **self.gov_headers) self.assertEqual(response.status_code, status.HTTP_201_CREATED) - self.assertEqual(models.DenialEntity.objects.count(), 5) + self.assertEqual(models.DenialEntity.objects.count(), self.total_denials) @pytest.mark.xfail(reason="This test is flaky and should be rewritten") # Occasionally causes this error: @@ -45,7 +48,7 @@ def test_adding_denials_to_application(self): def test_revoke_denial_without_comment_failure(self): response = self.client.get(reverse("external_data:denial-list"), **self.gov_headers) self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response.json()["count"], 5) + self.assertEqual(response.json()["count"], self.total_denials) denials = response.json()["results"] @@ -62,7 +65,7 @@ def test_revoke_denial_without_comment_failure(self): def test_revoke_denial_success(self): response = self.client.get(reverse("external_data:denial-list"), **self.gov_headers) self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response.json()["count"], 5) + self.assertEqual(response.json()["count"], self.total_denials) denials = response.json()["results"] @@ -82,7 +85,7 @@ def test_revoke_denial_success(self): def test_revoke_denial_active_success(self): response = self.client.get(reverse("external_data:denial-list"), **self.gov_headers) self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response.json()["count"], 5) + self.assertEqual(response.json()["count"], self.total_denials) denials = response.json()["results"] diff --git a/api/core/search/filter_backends.py b/api/core/search/filter_backends.py index e6ed977c83..8e64fd0b74 100644 --- a/api/core/search/filter_backends.py +++ b/api/core/search/filter_backends.py @@ -13,6 +13,20 @@ class QueryStringSearchFilterBackend(BaseSearchFilterBackend): search_param = "search" + def get_search_query_params(self, request): + """Get search query params. + + :param request: Django REST framework request. + :type request: rest_framework.request.Request + :return: List of search query params. + :rtype: list + """ + query_params = request.query_params.copy() + # This is required as query_string is unable to handle a single / + query_string = query_params.get(self.search_param, "").replace("/", "//") + query_params[self.search_param] = query_string + return query_params.getlist(self.search_param, []) + query_backends = [ QueryStringQueryBackend, ] diff --git a/api/core/search/validators.py b/api/core/search/validators.py index 0c9fe38206..980f80e293 100644 --- a/api/core/search/validators.py +++ b/api/core/search/validators.py @@ -12,6 +12,9 @@ def validate_search_terms(self): query_params = self.request.GET.copy() search_term = query_params.get("search") + # This is required as query_string is unable to handle a single / + search_term = search_term.replace("/", "//") + # Validation is only required if we are using QueryStringSearchFilterBackend if filter_backends.QueryStringSearchFilterBackend not in self.filter_backends: diff --git a/api/external_data/tests/denial_valid.csv b/api/external_data/tests/denial_valid.csv index 546972d534..032c5002c3 100644 --- a/api/external_data/tests/denial_valid.csv +++ b/api/external_data/tests/denial_valid.csv @@ -4,3 +4,5 @@ DN2000/0010,AB-CD-EF-300,Organisation Name 3,"2001 Street Name, City Name 3",Cou ,AB-XY-EF-900,The Widget Company,"2 Example Road, Example City",Example Country,Country Name X,"catch all",Extra Large Size Widget,Used in unknown industry,Risk of outcome 4,end_user DN3000/0000,AB-CD-EF-100,Organisation Name XYZ,"2000 Street Name, City Name 2",Country Name 2,Country Name 2,0A00200,Large Size Widget,Used in other industry,Risk of outcome 2,third_party DN4000/0000,AB-CD-EF-200,UK Issued,"2000 main road, some place",United Kingdom,Country Name 3,0A00300,Large Size Widget,Used in other industry,Risk of outcome 2,third_party +DN4102/0001,AB-CD-EF-400,Forward slash,"30/1 ltd",Country Name 4,Country Name 4,0A00504,Large Size Widget,Used in other industry,Risk of outcome 2,third_party +DN4103/0001,AB-CD-EF-500,c/o ltd,"forward slash",Country Name 5,Country Name 6,0A0050,Large Size Widget,Used in other industry,Risk of outcome 2,third_party diff --git a/api/external_data/tests/test_views.py b/api/external_data/tests/test_views.py index 9649641282..b99e73dfbf 100644 --- a/api/external_data/tests/test_views.py +++ b/api/external_data/tests/test_views.py @@ -9,7 +9,7 @@ from django.conf import settings from django.urls import reverse -from api.external_data import documents, models, serializers +from api.external_data import models, serializers from test_helpers.clients import DataTestClient denial_data_fields = [ @@ -24,6 +24,14 @@ class DenialViewSetTests(DataTestClient): + + def setUp(self): + super().setUp() + self.application = self.create_standard_application_case(self.organisation) + file_path = os.path.join(settings.BASE_DIR, "external_data/tests/denial_valid.csv") + with open(file_path, "rb") as f: + self.CSV_DENIAL_COUNT = len(f.readlines()) - 1 + def test_create_success(self): url = reverse("external_data:denial-list") file_path = os.path.join(settings.BASE_DIR, "external_data/tests/denial_valid.csv") @@ -32,8 +40,8 @@ def test_create_success(self): response = self.client.post(url, {"csv_file": content}, **self.gov_headers) self.assertEqual(response.status_code, 201) - self.assertEqual(models.DenialEntity.objects.count(), 5) - self.assertEqual(models.Denial.objects.count(), 5) + self.assertEqual(models.DenialEntity.objects.count(), self.CSV_DENIAL_COUNT) + self.assertEqual(models.Denial.objects.count(), self.CSV_DENIAL_COUNT) self.assertEqual( list( models.DenialEntity.objects.values( @@ -71,6 +79,18 @@ def test_create_success(self): "country": "Country Name 3", "entity_type": "third_party", }, + { + "name": "Forward slash", + "address": "30/1 ltd", + "country": "Country Name 4", + "entity_type": "third_party", + }, + { + "name": "c/o ltd", + "address": "forward slash", + "country": "Country Name 6", + "entity_type": "third_party", + }, ], ) self.assertEqual( @@ -121,6 +141,24 @@ def test_create_success(self): "end_use": "Used in other industry", "reason_for_refusal": "Risk of outcome 2", }, + { + "reference": "DN4102/0001", + "regime_reg_ref": "AB-CD-EF-400", + "notifying_government": "Country Name 4", + "denial_cle": "0A00504", + "item_description": "Large Size Widget", + "end_use": "Used in other industry", + "reason_for_refusal": "Risk of outcome 2", + }, + { + "reference": "DN4103/0001", + "regime_reg_ref": "AB-CD-EF-500", + "notifying_government": "Country Name 5", + "denial_cle": "0A0050", + "item_description": "Large Size Widget", + "end_use": "Used in other industry", + "reason_for_refusal": "Risk of outcome 2", + }, ], ) @@ -315,6 +353,14 @@ def test_create_sanitise_csv(self): class DenialSearchViewTests(DataTestClient): + + def setUp(self): + super().setUp() + self.application = self.create_standard_application_case(self.organisation) + file_path = os.path.join(settings.BASE_DIR, "external_data/tests/denial_valid.csv") + with open(file_path, "rb") as f: + self.CSV_DENIAL_COUNT = len(f.readlines()) - 1 + @pytest.mark.elasticsearch @parameterized.expand( [ @@ -330,7 +376,7 @@ def test_populate_denial_entity_objects(self, page_query): content = f.read() response = self.client.post(url, {"csv_file": content}, **self.gov_headers) self.assertEqual(response.status_code, 201) - self.assertEqual(models.DenialEntity.objects.count(), 5) + self.assertEqual(models.DenialEntity.objects.count(), self.CSV_DENIAL_COUNT) # Set one of them as revoked denial_entity = models.DenialEntity.objects.get(name="Organisation Name") @@ -398,8 +444,16 @@ def test_search_highlighting(self, search_query, expected_result): ({"search": "address:(Example)"}, ["AB-XY-EF-900"]), ({"search": "name:(UK Issued)"}, []), ({"search": "denial_cle:(catch all)"}, ["AB-XY-EF-900"]), - ({"search": "name:(Widget) OR address:(2001)"}, ["AB-CD-EF-300", "AB-XY-EF-900"]), + ( + {"search": "name:(Widget) OR address:(2001)"}, + [ + "AB-XY-EF-900", + "AB-CD-EF-300", + ], + ), ({"search": "name:(Organisation) AND address:(2000)"}, ["AB-CD-EF-100"]), + ({"search": "address:(30/1)"}, ["AB-CD-EF-400"]), + ({"search": "name:(c/o)"}, ["AB-CD-EF-500"]), ] ) def test_denial_entity_search(self, query, expected_items): @@ -418,7 +472,7 @@ def test_denial_entity_search(self, query, expected_items): self.assertEqual(response.status_code, 200) response_json = response.json() regime_reg_ref_results = [r["regime_reg_ref"] for r in response_json["results"]] - self.assertEqual(regime_reg_ref_results, expected_items) + self.assertListEqual(regime_reg_ref_results, expected_items) @pytest.mark.elasticsearch @parameterized.expand( diff --git a/api/search/product/tests/test_helpers.py b/api/search/product/tests/test_helpers.py index dcb8af9874..cc280c0fee 100644 --- a/api/search/product/tests/test_helpers.py +++ b/api/search/product/tests/test_helpers.py @@ -49,7 +49,7 @@ { "good": { "name": "Thermal camera", - "part_number": "IMG-1300", + "part_number": "IMG/1300", "is_good_controlled": True, "control_list_entries": ["6A005"], }, diff --git a/api/search/product/tests/test_views.py b/api/search/product/tests/test_views.py index 12d556d072..e344536efa 100644 --- a/api/search/product/tests/test_views.py +++ b/api/search/product/tests/test_views.py @@ -110,7 +110,8 @@ def test_product_search_by_name(self, query, expected_count, expected_name): [ ({"search": "ABC"}, 0), ({"search": "ABC-123"}, 1), - ({"search": "IMG-1300"}, 1), + ({"search": "IMG/1300"}, 1), + ({"search": "IMG-1300"}, 0), ({"search": "867-"}, 0), ({"search": "867-5309"}, 1), ({"search": "H2SO4"}, 1), From 834182f2a8653e4ec093fdecfeb09f20a5276d1a Mon Sep 17 00:00:00 2001 From: Arun Siluvery Date: Wed, 3 Jul 2024 07:41:31 +0100 Subject: [PATCH 55/73] Add dedicated endpoint to archive/restore a good Exporter has a choice to archive/restore a good and this is only allowed for goods that are submitted/verified on an application. However we have a restriction that goods cannot be edited once they are submitted on an application. Removing this restriction can cause various issues so to avoid this add a dedicated endpoint which only allows archiving/restoring a particular good. Add tests to ensure that only archive status can be updated using this endpoint. --- api/goods/serializers.py | 7 +++++++ api/goods/tests/test_edit.py | 35 +++++++++++++++++++++++++++++++++-- api/goods/urls.py | 3 ++- api/goods/views.py | 9 ++++++++- 4 files changed, 50 insertions(+), 4 deletions(-) diff --git a/api/goods/serializers.py b/api/goods/serializers.py index 687bfc095f..679a74d677 100644 --- a/api/goods/serializers.py +++ b/api/goods/serializers.py @@ -847,6 +847,13 @@ class Meta: ) +class GoodArchiveRestoreSerializer(serializers.ModelSerializer): + + class Meta: + model = Good + fields = ("is_archived",) + + class GoodArchiveHistorySerializer(serializers.Serializer): is_archived = serializers.SerializerMethodField() actioned_on = serializers.SerializerMethodField() diff --git a/api/goods/tests/test_edit.py b/api/goods/tests/test_edit.py index 593d488dc1..6e6c228003 100644 --- a/api/goods/tests/test_edit.py +++ b/api/goods/tests/test_edit.py @@ -572,7 +572,7 @@ def test_edit_design_details_field_success(self): def test_edit_archive_status(self): good = GoodFactory(organisation=self.organisation, item_category=ItemCategory.GROUP1_COMPONENTS) - url = reverse("goods:good_details", kwargs={"pk": str(good.id)}) + url = reverse("goods:archive_restore", kwargs={"pk": str(good.id)}) for version_count, is_archived in enumerate([True, False], start=1): request_data = {"is_archived": is_archived} @@ -580,7 +580,7 @@ def test_edit_archive_status(self): response = self.client.put(url, request_data, **self.exporter_headers) self.assertEqual(response.status_code, status.HTTP_200_OK) response = response.json() - self.assertEqual(response["good"]["is_archived"], is_archived) + self.assertEqual(response["is_archived"], is_archived) good.refresh_from_db() self.assertEqual(good.is_archived, is_archived) @@ -593,6 +593,37 @@ def test_edit_archive_status(self): for version in versions: self.assertEqual(version.revision.user, self.exporter_user.baseuser_ptr) + @parameterized.expand( + [ + [ + {"name": "Rifle", "is_archived": None}, + {"is_archived": True}, + {"name": "Rifle", "is_archived": True}, + ], + [ + {"name": "Rifle", "is_archived": True}, + {"is_archived": False}, + {"name": "Rifle", "is_archived": False}, + ], + [ + {"name": "Rifle", "is_archived": None}, + {"name": "Rifle updated", "is_archived": False}, + {"name": "Rifle", "is_archived": False}, + ], + ] + ) + def test_only_archive_field_can_be_updated(self, initial, data, expected): + good = GoodFactory(organisation=self.organisation, **initial) + url = reverse("goods:archive_restore", kwargs={"pk": str(good.id)}) + + response = self.client.put(url, data, **self.exporter_headers) + self.assertEqual(response.status_code, status.HTTP_200_OK) + response = response.json() + + good.refresh_from_db() + actual = {"name": good.name, "is_archived": good.is_archived} + self.assertEqual(actual, expected) + class GoodsAttachingTests(DataTestClient): def setUp(self): diff --git a/api/goods/urls.py b/api/goods/urls.py index 554e223f4c..36eab53aa9 100644 --- a/api/goods/urls.py +++ b/api/goods/urls.py @@ -6,7 +6,6 @@ urlpatterns = [ path("", views.GoodList.as_view(), name="goods"), - path("archived-goods/", views.ArchivedGoodList.as_view(), name="archived_goods"), path("/", views.GoodOverview.as_view(), name="good"), path("/attaching/", views.GoodAttaching.as_view(), name="good_attaching"), path("/details/", views.GoodTAUDetails.as_view(), name="good_details"), @@ -39,4 +38,6 @@ views.DocumentGoodOnApplicationInternalDetailView.as_view(), name="document_internal_good_on_application_detail", ), + path("archived-goods/", views.ArchivedGoodList.as_view(), name="archived_goods"), + path("/archive-restore/", views.GoodArchiveRestore.as_view(), name="archive_restore"), ] diff --git a/api/goods/views.py b/api/goods/views.py index 1b204d187e..c4e010dd08 100644 --- a/api/goods/views.py +++ b/api/goods/views.py @@ -4,7 +4,7 @@ from django.http import JsonResponse from django.shortcuts import get_object_or_404 from rest_framework import status -from rest_framework.generics import ListAPIView, ListCreateAPIView +from rest_framework.generics import ListAPIView, ListCreateAPIView, UpdateAPIView from rest_framework.views import APIView from api.applications.models import ( @@ -46,6 +46,7 @@ GoodDocumentAvailabilitySerializer, GoodDocumentSensitivitySerializer, TinyGoodDetailsSerializer, + GoodArchiveRestoreSerializer, ) from api.applications.serializers.good import ( GoodOnApplicationInternalDocumentCreateSerializer, @@ -194,6 +195,12 @@ def get_queryset(self): return queryset.order_by("-updated_at") +class GoodArchiveRestore(UpdateAPIView): + authentication_classes = (ExporterAuthentication,) + queryset = Good.objects.all() + serializer_class = GoodArchiveRestoreSerializer + + class GoodDocumentAvailabilityCheck(APIView): """ Check document is attached to application good/product From d4769a799999c307b45e87aff575c0212d3cdc57 Mon Sep 17 00:00:00 2001 From: Kevin Carrogan Date: Wed, 3 Jul 2024 10:27:52 +0100 Subject: [PATCH 56/73] Fix issue with failing parent filter test --- api/core/tests/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/api/core/tests/views.py b/api/core/tests/views.py index 75964d3f84..d5bbd7a9f4 100644 --- a/api/core/tests/views.py +++ b/api/core/tests/views.py @@ -12,6 +12,7 @@ class MisconfiguredParentFilterView(RetrieveAPIView): class ParentFilterView(RetrieveAPIView): filter_backends = (ParentFilter,) + lookup_url_kwarg = "child_pk" parent_filter_id_lookup_field = "parent_id" queryset = ChildModel.objects.all() serializer_class = ChildModelSerializer From d412587c0334e83ef50663331c6b34b42f7e30ee Mon Sep 17 00:00:00 2001 From: Kevin Carrogan Date: Wed, 3 Jul 2024 10:31:41 +0100 Subject: [PATCH 57/73] Remove database fields for readonly and terminal statuses This is to stop the situations where we may have a discrepancy between what's in the database and what's in the code This switches it so that the code is the only way of marking these attributes on a status --- api/cases/managers.py | 9 ++++-- api/staticdata/management/SeedCommand.py | 12 ++++++-- .../management/commands/seedcasestatuses.py | 2 +- .../management/commands/seedcountries.py | 2 +- ...remove_casestatus_is_read_only_and_more.py | 21 ++++++++++++++ api/staticdata/statuses/models.py | 28 +++++++++++++++++-- api/staticdata/statuses/views.py | 9 ++++-- 7 files changed, 71 insertions(+), 12 deletions(-) create mode 100644 api/staticdata/statuses/migrations/0014_remove_casestatus_is_read_only_and_more.py diff --git a/api/cases/managers.py b/api/cases/managers.py index b8ec201e3e..6a48c290ed 100644 --- a/api/cases/managers.py +++ b/api/cases/managers.py @@ -6,6 +6,11 @@ from django.db.models import Prefetch, Q, Sum from django.utils import timezone +from queryable_properties.managers import ( + QueryablePropertiesManager, + QueryablePropertiesQuerySet, +) + from api.cases.enums import AdviceLevel, CaseTypeEnum from api.cases.helpers import get_updated_case_ids, get_assigned_to_user_case_ids, get_assigned_as_case_officer_case_ids from api.common.enums import SortOrder @@ -24,7 +29,7 @@ from api.staticdata.statuses.libraries.get_case_status import get_case_status_by_status -class CaseQuerySet(models.QuerySet): +class CaseQuerySet(QueryablePropertiesQuerySet): """ Custom queryset for the Case model. This allows us to chain application specific filtering logic in a reusable way. @@ -236,7 +241,7 @@ def includes_refusal_recommendation_from_ogd(self): return self.filter(advice__type=AdviceType.REFUSE, advice__user__team__is_ogd=True) -class CaseManager(models.Manager): +class CaseManager(QueryablePropertiesManager): """ Custom manager for the Case model that uses CaseQuerySet and provides a reusable search functionality to the Case model. diff --git a/api/staticdata/management/SeedCommand.py b/api/staticdata/management/SeedCommand.py index 8d9a9e7baf..47cead3ef8 100644 --- a/api/staticdata/management/SeedCommand.py +++ b/api/staticdata/management/SeedCommand.py @@ -69,7 +69,7 @@ def read_csv(filename: str): return list(reader) @staticmethod - def update_or_create(model: models.Model, rows: list): + def update_or_create(model: models.Model, rows: list, exclude=None): """ Takes a list of dicts with an id field and other properties applicable to a given model. If an object with the given id exists, it will update all @@ -77,20 +77,26 @@ def update_or_create(model: models.Model, rows: list): :param model: A given Django model to populate :param rows: A list of dictionaries (csv entries) to populate to the model """ + exclude = exclude or [] for row in rows: obj_id = row["id"] obj = model.objects.filter(id=obj_id) if not obj.exists(): + for key in exclude: + del row[key] model.objects.create(**row) if not settings.SUPPRESS_TEST_OUTPUT: print(f"CREATED {model.__name__}: {dict(row)}") else: - SeedCommand.update_if_not_equal(obj, row) + SeedCommand.update_if_not_equal(obj, row, exclude) @staticmethod - def update_if_not_equal(obj: QuerySet, row: dict): + def update_if_not_equal(obj: QuerySet, row: dict, exclude=None): # Can not delete the "id" key-value from `rows` as it will manipulate the data which is later used in # `delete_unused_objects` + exclude = exclude or [] + for key in exclude: + del row[key] attributes = {k: v for k, v in row.items() if k != "id"} obj = obj.exclude(**attributes) if obj.exists(): diff --git a/api/staticdata/management/commands/seedcasestatuses.py b/api/staticdata/management/commands/seedcasestatuses.py index fe7567377b..bb027f3466 100644 --- a/api/staticdata/management/commands/seedcasestatuses.py +++ b/api/staticdata/management/commands/seedcasestatuses.py @@ -46,7 +46,7 @@ def operation(self, *args, **options): if row["workflow_sequence"] == "None": row["workflow_sequence"] = None - self.update_or_create(CaseStatus, status_csv) + self.update_or_create(CaseStatus, status_csv, exclude=["is_terminal", "is_read_only"]) case_type_list = CaseTypeEnum.CASE_TYPE_LIST diff --git a/api/staticdata/management/commands/seedcountries.py b/api/staticdata/management/commands/seedcountries.py index e15319eb30..049d1533bf 100644 --- a/api/staticdata/management/commands/seedcountries.py +++ b/api/staticdata/management/commands/seedcountries.py @@ -28,7 +28,7 @@ def operation(self, *args, **options): if not settings.SUPPRESS_TEST_OUTPUT: print(f"CREATED {Country.__name__}: {dict(row)}") else: - SeedCommand.update_if_not_equal(obj, row) + SeedCommand.update_if_not_equal(obj, row, exclude=["is_terminal", "is_read_only"]) ids = [row["id"] for row in csv] for obj in Country.objects.all(): diff --git a/api/staticdata/statuses/migrations/0014_remove_casestatus_is_read_only_and_more.py b/api/staticdata/statuses/migrations/0014_remove_casestatus_is_read_only_and_more.py new file mode 100644 index 0000000000..57f14c12e9 --- /dev/null +++ b/api/staticdata/statuses/migrations/0014_remove_casestatus_is_read_only_and_more.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.13 on 2024-07-01 16:15 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("statuses", "0013_add_superseded_by_amendment_status"), + ] + + operations = [ + migrations.RemoveField( + model_name="casestatus", + name="is_read_only", + ), + migrations.RemoveField( + model_name="casestatus", + name="is_terminal", + ), + ] diff --git a/api/staticdata/statuses/models.py b/api/staticdata/statuses/models.py index a4c9fbdce0..d6009eb18c 100644 --- a/api/staticdata/statuses/models.py +++ b/api/staticdata/statuses/models.py @@ -1,9 +1,15 @@ import uuid from django.db import models +from django.db.models import Q +from queryable_properties.managers import QueryablePropertiesManager +from queryable_properties.properties import queryable_property -class CaseStatusManager(models.Manager): +from api.staticdata.statuses.enums import CaseStatusEnum + + +class CaseStatusManager(QueryablePropertiesManager): def get_by_natural_key(self, status): return self.get(status=status) @@ -13,12 +19,28 @@ class CaseStatus(models.Model): status = models.CharField(max_length=50, unique=True) priority = models.PositiveSmallIntegerField(null=False, blank=False) workflow_sequence = models.PositiveSmallIntegerField(null=True) - is_read_only = models.BooleanField(blank=False, null=True) - is_terminal = models.BooleanField(blank=False, null=True) next_workflow_status = models.ForeignKey("CaseStatus", on_delete=models.DO_NOTHING, null=True, blank=True) objects = CaseStatusManager() + @queryable_property + def is_terminal(self): + return CaseStatusEnum.is_terminal(self.status) + + @is_terminal.filter(boolean=True) + @classmethod + def is_terminal(cls): + return Q(status__in=CaseStatusEnum.terminal_statuses()) + + @queryable_property + def is_read_only(self): + return CaseStatusEnum.is_read_only(self.status) + + @is_read_only.filter(boolean=True) + @classmethod + def is_read_only(cls): + return Q(status__in=CaseStatusEnum.read_only_statuses()) + def natural_key(self): return (self.status,) diff --git a/api/staticdata/statuses/views.py b/api/staticdata/statuses/views.py index 7ff7b60316..c3584753da 100644 --- a/api/staticdata/statuses/views.py +++ b/api/staticdata/statuses/views.py @@ -4,6 +4,7 @@ from api.cases.views.search.service import get_case_status_list from api.core.authentication import SharedAuthentication +from api.staticdata.statuses.enums import CaseStatusEnum from api.staticdata.statuses.models import CaseStatus @@ -20,7 +21,11 @@ class StatusProperties(APIView): def get(self, request, status): """Return is_read_only and is_terminal properties for a case status.""" - status_properties = CaseStatus.objects.filter(status=status).values_list("is_read_only", "is_terminal")[0] + case_status = CaseStatus.objects.get(status=status) return JsonResponse( - data={"is_read_only": status_properties[0], "is_terminal": status_properties[1]}, status=HTTP_200_OK + data={ + "is_read_only": CaseStatusEnum.is_read_only(case_status), + "is_terminal": CaseStatusEnum.is_terminal(case_status), + }, + status=HTTP_200_OK, ) From d6ea7bc64c1a72f7fa99568e8c0f5a54e4216097 Mon Sep 17 00:00:00 2001 From: Kevin Carrogan Date: Wed, 3 Jul 2024 10:33:58 +0100 Subject: [PATCH 58/73] Flip readonly statuses to be writeable by inclusion and add explicit major editable statuses The statuses for readonly was potentially open to someone forgetting to add a new status and inadvertantly having a status be writable without that being the correct intention This flips the logic around so that all statuses are readonly by default until explicitly set otherwise --- .../libraries/application_helpers.py | 3 +- api/applications/tests/test_adding_sites.py | 9 ++- .../tests/test_application_status.py | 9 ++- .../tests/test_external_locations.py | 8 +-- .../tests/test_matching_denials.py | 4 ++ api/applications/tests/test_removing_goods.py | 26 +++---- api/applications/views/denials.py | 5 +- api/core/tests/test_decorators.py | 9 ++- .../tests/test_gov_user_notifications.py | 3 +- api/staticdata/statuses/enums.py | 69 +++++++------------ test_helpers/clients.py | 4 ++ 11 files changed, 73 insertions(+), 76 deletions(-) diff --git a/api/applications/libraries/application_helpers.py b/api/applications/libraries/application_helpers.py index 79251f2c1b..ca0f1025af 100644 --- a/api/applications/libraries/application_helpers.py +++ b/api/applications/libraries/application_helpers.py @@ -38,6 +38,8 @@ def can_status_be_set_by_exporter_user(original_status: str, new_status: str) -> elif new_status == CaseStatusEnum.SURRENDERED: if original_status != CaseStatusEnum.FINALISED: return False + elif CaseStatusEnum.can_invoke_major_edit(original_status) and new_status == CaseStatusEnum.APPLICANT_EDITING: + return True elif CaseStatusEnum.is_read_only(original_status) or new_status != CaseStatusEnum.APPLICANT_EDITING: return False @@ -89,7 +91,6 @@ def check_user_can_set_status(request, application, data): Checks whether an user (internal/exporter) can set the requested status Returns error response if user cannot set the status, None otherwise """ - if hasattr(request.user, "exporteruser"): if get_request_user_organisation_id(request) != application.organisation.id: raise PermissionDenied() diff --git a/api/applications/tests/test_adding_sites.py b/api/applications/tests/test_adding_sites.py index 73a018bf91..42e0879990 100644 --- a/api/applications/tests/test_adding_sites.py +++ b/api/applications/tests/test_adding_sites.py @@ -97,8 +97,7 @@ def test_adding_site_to_draft_deletes_external_locations(self): self.assertEqual(SiteOnApplication.objects.filter(application=draft).count(), 1) self.assertEqual(ExternalLocationOnApplication.objects.filter(application=draft).count(), 0) - def test_add_site_to_a_submitted_application_success(self): - + def test_add_site_to_a_submitted_application_failure(self): site_to_add = SiteFactory(organisation=self.organisation, address=AddressFactoryGB()) data = {"sites": [self.primary_site.id, site_to_add.id]} @@ -107,10 +106,10 @@ def test_add_site_to_a_submitted_application_success(self): response = self.client.post(self.url, data, **self.exporter_headers) self.application.refresh_from_db() - self.assertEqual(response.status_code, status.HTTP_201_CREATED) - self.assertEqual(SiteOnApplication.objects.filter(application=self.application).count(), 2) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(SiteOnApplication.objects.filter(application=self.application).count(), 1) - def test_add_site_to_a_submitted_application_failure(self): + def test_add_site_to_a_submitted_application_failure_different_country(self): """ Cannot add additional site to a submitted application unless the additional site is located in a country that is already on the application diff --git a/api/applications/tests/test_application_status.py b/api/applications/tests/test_application_status.py index 66755e3c0e..7fd884d85d 100644 --- a/api/applications/tests/test_application_status.py +++ b/api/applications/tests/test_application_status.py @@ -57,6 +57,8 @@ def test_set_application_status_on_application_not_in_users_organisation_failure @mock.patch("api.applications.views.applications.notify_exporter_case_opened_for_editing") def test_exporter_set_application_status_applicant_editing_when_in_editable_status_success(self, mock_notify): self.submit_application(self.standard_application) + self.standard_application.status = get_case_status_by_status(CaseStatusEnum.REOPENED_FOR_CHANGES) + self.standard_application.save() data = {"status": CaseStatusEnum.APPLICANT_EDITING} response = self.client.put(self.url, data=data, **self.exporter_headers) @@ -68,7 +70,8 @@ def test_exporter_set_application_status_applicant_editing_when_in_editable_stat audit_event = Audit.objects.first() self.assertEqual(audit_event.verb, AuditType.UPDATED_STATUS) self.assertEqual( - audit_event.payload, {"status": {"new": CaseStatusEnum.APPLICANT_EDITING, "old": CaseStatusEnum.SUBMITTED}} + audit_event.payload, + {"status": {"new": CaseStatusEnum.APPLICANT_EDITING, "old": CaseStatusEnum.REOPENED_FOR_CHANGES}}, ) mock_notify.assert_called_with(self.standard_application) @@ -413,7 +416,9 @@ def test_hmrc_user_set_status_to_finalised_fail(self, mock_authenticate): self.standard_application.save() self.assertEqual(self.standard_application.status, get_case_status_by_status(CaseStatusEnum.UNDER_FINAL_REVIEW)) - base_user = BaseUser(email="test@mail.com", first_name="John", last_name="Smith", type=UserType.SYSTEM) + base_user = BaseUser( + email="test@mail.com", first_name="John", last_name="Smith", type=UserType.SYSTEM # /PS-IGNORE + ) base_user.team = Team.objects.get(id=TeamIdEnum.LICENSING_UNIT) base_user.save() diff --git a/api/applications/tests/test_external_locations.py b/api/applications/tests/test_external_locations.py index a661bfe7a0..f9eb4fd380 100644 --- a/api/applications/tests/test_external_locations.py +++ b/api/applications/tests/test_external_locations.py @@ -43,7 +43,7 @@ def test_adding_external_location_to_unsubmitted_application_removes_sites(self) 1, ) - def test_add_external_location_to_a_submitted_application_success(self): + def test_add_external_location_to_a_submitted_application_failure(self): SiteOnApplication.objects.filter(application=self.application).delete() ExternalLocationOnApplication(application=self.application, external_location=self.external_location).save() external_location_to_add = self.create_external_location("storage facility 2", self.organisation) @@ -58,13 +58,13 @@ def test_add_external_location_to_a_submitted_application_success(self): response = self.client.post(self.url, data, **self.exporter_headers) self.application.refresh_from_db() - self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) self.assertEqual( ExternalLocationOnApplication.objects.filter(application=self.application).count(), - 2, + 1, ) - def test_add_external_location_to_a_submitted_application_failure(self): + def test_add_external_location_already_on_application_to_a_submitted_application_failure(self): """ Cannot add additional external locations to a submitted application unless the additional external location is located in a country that is already on the application diff --git a/api/applications/tests/test_matching_denials.py b/api/applications/tests/test_matching_denials.py index 0a59a460ca..cb519e81fd 100644 --- a/api/applications/tests/test_matching_denials.py +++ b/api/applications/tests/test_matching_denials.py @@ -7,6 +7,8 @@ from api.applications.tests.factories import DenialEntityFactory from api.external_data import models +from api.staticdata.statuses.enums import CaseStatusEnum +from api.staticdata.statuses.libraries.get_case_status import get_case_status_by_status from test_helpers.clients import DataTestClient @@ -14,6 +16,8 @@ class ApplicationDenialMatchesOnApplicationTests(DataTestClient): def setUp(self): super().setUp() self.application = self.create_standard_application_case(self.organisation) + self.application.status = get_case_status_by_status(CaseStatusEnum.INITIAL_CHECKS) + self.application.save() file_path = os.path.join(settings.BASE_DIR, "external_data/tests/denial_valid.csv") with open(file_path, "rb") as f: content = f.read() diff --git a/api/applications/tests/test_removing_goods.py b/api/applications/tests/test_removing_goods.py index e56d2e5dd9..49d561507e 100644 --- a/api/applications/tests/test_removing_goods.py +++ b/api/applications/tests/test_removing_goods.py @@ -10,6 +10,7 @@ from api.goods.enums import GoodStatus from api.goods.models import Good, FirearmGoodDetails from api.users.models import UserOrganisationRelationship +from api.staticdata.statuses.enums import CaseStatusEnum from api.staticdata.statuses.libraries.get_case_status import get_case_status_by_status from api.staticdata.units.enums import Units from test_helpers.clients import DataTestClient @@ -26,10 +27,7 @@ def test_remove_a_good_from_draft_success(self): Then the good_on_application is deleted And the good status is changed to DRAFT """ - draft = self.create_draft_standard_application(self.organisation) - application = self.submit_application( - draft - ) # This will submit the application and set the good status to SUBMITTED + application = self.create_draft_standard_application(self.organisation) good_on_application = application.goods.first() url = reverse( @@ -40,7 +38,7 @@ def test_remove_a_good_from_draft_success(self): response = self.client.delete(url, **self.exporter_headers) self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(GoodOnApplication.objects.filter(application=draft).count(), 0) + self.assertEqual(GoodOnApplication.objects.filter(application=application).count(), 0) self.assertEqual(good_on_application.good.status, GoodStatus.DRAFT) def test_remove_a_good_from_draft_success_when_good_is_verified(self): @@ -52,8 +50,7 @@ def test_remove_a_good_from_draft_success_when_good_is_verified(self): Then the good_on_application is deleted And the good status is not changed """ - draft = self.create_draft_standard_application(self.organisation) - application = self.submit_application(draft) + application = self.create_draft_standard_application(self.organisation) good_on_application = application.goods.first() good_on_application.good.status = GoodStatus.VERIFIED good_on_application.good.save() @@ -66,7 +63,7 @@ def test_remove_a_good_from_draft_success_when_good_is_verified(self): response = self.client.delete(url, **self.exporter_headers) self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(GoodOnApplication.objects.filter(application=draft).count(), 0) + self.assertEqual(GoodOnApplication.objects.filter(application=application).count(), 0) self.assertEqual(good_on_application.good.status, GoodStatus.VERIFIED) def test_remove_a_good_from_application_success_when_good_is_on_multiple_applications( @@ -82,6 +79,7 @@ def test_remove_a_good_from_application_success_when_good_is_on_multiple_applica """ application1 = self.create_draft_standard_application(self.organisation) self.submit_application(application1) + self.set_application_status(application1, CaseStatusEnum.APPLICANT_EDITING) good_on_application1 = GoodOnApplication.objects.get(application=application1) application2 = self.create_draft_standard_application(self.organisation) @@ -130,8 +128,7 @@ def test_remove_a_good_that_does_not_exist_from_draft(self): self.assertEqual(GoodOnApplication.objects.filter(application=draft).count(), 1) def test_remove_a_good_from_draft_as_gov_user_failure(self): - draft = self.create_draft_standard_application(self.organisation) - application = self.submit_application(draft, self.exporter_user) + application = self.create_draft_standard_application(self.organisation) good_on_application = application.goods.first() url = reverse( @@ -145,8 +142,7 @@ def test_remove_a_good_from_draft_as_gov_user_failure(self): self.assertEqual(GoodOnApplication.objects.filter(application=application).count(), 1) def test_remove_goods_from_application_not_in_users_organisation_failure(self): - draft = self.create_draft_standard_application(self.organisation) - application = self.submit_application(draft) + application = self.create_draft_standard_application(self.organisation) good_on_application = application.goods.first() url = reverse( "applications:good_on_application", @@ -167,8 +163,7 @@ def test_remove_goods_from_application_not_in_users_organisation_failure(self): @parameterized.expand(get_case_statuses(read_only=False)) def test_delete_good_from_application_in_an_editable_status_success(self, editable_status): application = self.create_draft_standard_application(self.organisation) - application.status = get_case_status_by_status(editable_status) - application.save() + self.set_application_status(application, editable_status) good_on_application = application.goods.first() url = reverse( "applications:good_on_application", @@ -183,8 +178,7 @@ def test_delete_good_from_application_in_an_editable_status_success(self, editab @parameterized.expand(get_case_statuses(read_only=True)) def test_delete_good_from_application_in_read_only_status_failure(self, read_only_status): application = self.create_draft_standard_application(self.organisation) - application.status = get_case_status_by_status(read_only_status) - application.save() + self.set_application_status(application, read_only_status) good_on_application = application.goods.first() url = reverse( "applications:good_on_application", diff --git a/api/applications/views/denials.py b/api/applications/views/denials.py index 1d50e6634c..181787ec35 100644 --- a/api/applications/views/denials.py +++ b/api/applications/views/denials.py @@ -3,10 +3,10 @@ from rest_framework import status from rest_framework.views import APIView -from api.applications.libraries.case_status_helpers import get_case_statuses from api.applications.models import BaseApplication, DenialMatchOnApplication from api.applications.serializers import denial from api.core.authentication import GovAuthentication +from api.staticdata.statuses.enums import CaseStatusEnum from lite_content.lite_api import strings @@ -25,7 +25,6 @@ def get(self, request, pk): return JsonResponse(data={"denial_matches": denial_matches_data}) def post(self, request, pk): - serializer = denial.DenialMatchOnApplicationCreateSerializer(data=request.data, many=True) if serializer.is_valid(): serializer.save() @@ -36,7 +35,7 @@ def post(self, request, pk): def delete(self, request, pk): application = get_object_or_404(BaseApplication.objects.all(), pk=pk) - if application.status.status in get_case_statuses(read_only=True): + if application.status.status != CaseStatusEnum.INITIAL_CHECKS: return JsonResponse( data={"errors": [strings.Applications.Generic.READ_ONLY]}, status=status.HTTP_400_BAD_REQUEST, diff --git a/api/core/tests/test_decorators.py b/api/core/tests/test_decorators.py index dbd099adea..6dfe481b2d 100644 --- a/api/core/tests/test_decorators.py +++ b/api/core/tests/test_decorators.py @@ -1,13 +1,17 @@ +from parameterized import parameterized + from django.http import HttpResponse from django.test import RequestFactory from rest_framework import status +from api.applications.libraries.case_status_helpers import get_case_statuses from api.cases.enums import CaseTypeSubTypeEnum from api.core.authentication import ORGANISATION_ID from api.core.decorators import allowed_application_types, application_in_state, authorised_to_view_application from lite_content.lite_api import strings from api.organisations.tests.factories import OrganisationFactory from api.staticdata.statuses.enums import CaseStatusEnum +from api.staticdata.statuses.libraries.get_case_status import get_case_status_by_status from api.staticdata.statuses.models import CaseStatus from test_helpers.clients import DataTestClient from api.users.models import ExporterUser, GovUser @@ -42,8 +46,11 @@ def a_view(request, *args, **kwargs): self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST) self.assertTrue("This operation can only be used on applications of type:" in resp.content.decode("utf-8")) - def test_application_in_state_editable_success(self): + @parameterized.expand(get_case_statuses(read_only=False)) + def test_application_in_state_editable_success(self, editable_status): application = self.create_standard_application_case(self.organisation) + application.status = get_case_status_by_status(editable_status) + application.save() @application_in_state(is_editable=True) def a_view(request, *args, **kwargs): diff --git a/api/gov_users/tests/test_gov_user_notifications.py b/api/gov_users/tests/test_gov_user_notifications.py index 001bd884d8..c6ada47e9f 100644 --- a/api/gov_users/tests/test_gov_user_notifications.py +++ b/api/gov_users/tests/test_gov_user_notifications.py @@ -13,7 +13,8 @@ class GovUserNotificationTests(DataTestClient): def setUp(self): super().setUp() - self.case = self.create_standard_application_case(self.organisation, "Case") + self.case = self.create_draft_standard_application(self.organisation, "Case") + self.set_application_status(self.case, CaseStatusEnum.APPLICANT_EDITING) self.audit_content_type = ContentType.objects.get_for_model(Audit) def test_edit_application_creates_new_audit_notification_success(self): diff --git a/api/staticdata/statuses/enums.py b/api/staticdata/statuses/enums.py index 801b6ca62b..c057057c59 100644 --- a/api/staticdata/statuses/enums.py +++ b/api/staticdata/statuses/enums.py @@ -40,27 +40,13 @@ class CaseStatusEnum: _system_status = [DRAFT] - _read_only_statuses = [ - APPEAL_REVIEW, - APPEAL_FINAL_REVIEW, - CHANGE_UNDER_REVIEW, - CHANGE_UNDER_FINAL_REVIEW, - CLOSED, - DEREGISTERED, - FINALISED, - REGISTERED, - REOPENED_DUE_TO_ORG_CHANGES, - UNDER_ECJU_REVIEW, - UNDER_FINAL_REVIEW, - REVOKED, - SURRENDERED, - SUSPENDED, - WITHDRAWN, - OGD_ADVICE, - OGD_CONSOLIDATION, - SUPERSEDED_BY_AMENDMENT, + _writeable_statuses = [ + DRAFT, + APPLICANT_EDITING, ] + _can_invoke_major_edit_statuses = [SUBMITTED, INITIAL_CHECKS, UNDER_REVIEW, REOPENED_FOR_CHANGES] + _major_editable_statuses = [APPLICANT_EDITING, DRAFT] _terminal_statuses = [ @@ -125,6 +111,8 @@ class CaseStatusEnum: (OGD_ADVICE, "OGD Advice"), (OGD_CONSOLIDATION, "OGD Consolidation"), (SUPERSEDED_BY_AMENDMENT, "Superseded by amendment"), + (FINAL_REVIEW_COUNTERSIGN, "Final review countersign"), + (FINAL_REVIEW_SECOND_COUNTERSIGN, "Final review second countersign"), ] priority = { @@ -166,27 +154,7 @@ class CaseStatusEnum: @classmethod def get_choices(cls): - lu_countersign_statuses = [] - lu_countersign_statuses.extend( - [ - (cls.FINAL_REVIEW_COUNTERSIGN, "Final review countersign"), - (cls.FINAL_REVIEW_SECOND_COUNTERSIGN, "Final review second countersign"), - ] - ) - - return cls.choices + lu_countersign_statuses - - @classmethod - def get_read_only_choices(cls): - lu_countersign_statuses = [] - lu_countersign_statuses.extend( - [ - cls.FINAL_REVIEW_COUNTERSIGN, - cls.FINAL_REVIEW_SECOND_COUNTERSIGN, - ] - ) - - return cls._read_only_statuses + lu_countersign_statuses + return cls.choices @classmethod def get_text(cls, status): @@ -204,7 +172,7 @@ def get_value(cls, status): @classmethod def is_read_only(cls, status): - return status in cls.get_read_only_choices() + return status in cls.read_only_statuses() @classmethod def is_terminal(cls, status): @@ -216,12 +184,20 @@ def is_system_status(cls, status): @classmethod def read_only_statuses(cls): - return cls.get_read_only_choices() + return list(set(cls.all()) - set(cls._writeable_statuses)) @classmethod def major_editable_statuses(cls): return cls._major_editable_statuses + @classmethod + def can_invoke_major_edit_statuses(cls): + return cls._can_invoke_major_edit_statuses + + @classmethod + def can_invoke_major_edit(cls, status): + return status in cls._can_invoke_major_edit_statuses + @classmethod def terminal_statuses(cls): return cls._terminal_statuses @@ -237,7 +213,14 @@ def as_list(cls): @classmethod def all(cls): - return [k for k, _ in [*cls.get_choices(), (cls.DRAFT, "Draft")]] + _all = [] + for name, _type in CaseStatusEnum.__dict__.items(): + if not name.isupper(): + continue + if type(_type) is not str: + continue + _all.append(getattr(cls, name)) + return _all class CaseStatusIdEnum: diff --git a/test_helpers/clients.py b/test_helpers/clients.py index b754d3938a..8cc134a58e 100644 --- a/test_helpers/clients.py +++ b/test_helpers/clients.py @@ -938,6 +938,10 @@ def get_object_from_default_bucket(self, key): Key=key, ) + def set_application_status(self, application, status_name): + application.status = get_case_status_by_status(status_name) + application.save() + @pytest.mark.performance # we need to set debug to true otherwise we can't see the amount of queries From eec5a1b08fa95688d953ec53b8542672c535e758 Mon Sep 17 00:00:00 2001 From: Arun Siluvery Date: Wed, 3 Jul 2024 11:18:55 +0100 Subject: [PATCH 59/73] Explicitly specify the list of archivable statuses --- api/goods/enums.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api/goods/enums.py b/api/goods/enums.py index 08a9927d8a..01878a3cc9 100644 --- a/api/goods/enums.py +++ b/api/goods/enums.py @@ -14,6 +14,12 @@ class GoodStatus: (VERIFIED, "Verified"), ] + _archivable_statuses = [SUBMITTED, VERIFIED] + + @classmethod + def archivable_statuses(cls): + return cls._archivable_statuses + class ItemType: EQUIPMENT = "equipment" From 226ee3fb98d00a4eee06a32abb2a6f92f4858c3b Mon Sep 17 00:00:00 2001 From: Arun Siluvery Date: Wed, 3 Jul 2024 11:19:53 +0100 Subject: [PATCH 60/73] Ensure only goods from own organisation can be archive/restored --- api/goods/tests/test_views.py | 13 +++++++++++++ api/goods/views.py | 8 +++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/api/goods/tests/test_views.py b/api/goods/tests/test_views.py index a45af89412..bdf70320ae 100644 --- a/api/goods/tests/test_views.py +++ b/api/goods/tests/test_views.py @@ -228,3 +228,16 @@ def test_view_archived_goods_filter_by_control_list_entry(self, control_list_ent self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(len(response_data), count) + + def test_cannot_archive_good_from_other_organisation(self): + good = GoodFactory(organisation=self.organisation, status=GoodStatus.SUBMITTED) + url = reverse("goods:archive_restore", kwargs={"pk": str(good.id)}) + + headers = self.exporter_headers.copy() + headers["HTTP_ORGANISATION_ID"] = str(good.id) + + response = self.client.put(url, {"is_archived": True}, **headers) + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + good.refresh_from_db() + self.assertIsNone(good.is_archived) diff --git a/api/goods/views.py b/api/goods/views.py index c4e010dd08..47832eb5ca 100644 --- a/api/goods/views.py +++ b/api/goods/views.py @@ -197,9 +197,15 @@ def get_queryset(self): class GoodArchiveRestore(UpdateAPIView): authentication_classes = (ExporterAuthentication,) - queryset = Good.objects.all() serializer_class = GoodArchiveRestoreSerializer + def get_queryset(self): + organisation = get_request_user_organisation_id(self.request) + + return Good.objects.filter( + organisation=organisation, + ) + class GoodDocumentAvailabilityCheck(APIView): """ From a357a35b073d077ebec3cf5039dd66a6cc8dd62a Mon Sep 17 00:00:00 2001 From: Arun Siluvery Date: Wed, 3 Jul 2024 11:20:47 +0100 Subject: [PATCH 61/73] Ensure only goods with archivable status can be archive/restored --- api/goods/tests/test_edit.py | 42 ++++++++++++++++++++++++++++++++++-- api/goods/views.py | 1 + 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/api/goods/tests/test_edit.py b/api/goods/tests/test_edit.py index 6e6c228003..2754737b18 100644 --- a/api/goods/tests/test_edit.py +++ b/api/goods/tests/test_edit.py @@ -7,6 +7,7 @@ from reversion.models import Version from api.goods.enums import ( + GoodStatus, GoodPvGraded, PvGrading, MilitaryUse, @@ -571,7 +572,11 @@ def test_edit_design_details_field_success(self): self.assertEqual(good.design_details, "design details") def test_edit_archive_status(self): - good = GoodFactory(organisation=self.organisation, item_category=ItemCategory.GROUP1_COMPONENTS) + good = GoodFactory( + organisation=self.organisation, + status=GoodStatus.SUBMITTED, + item_category=ItemCategory.GROUP1_COMPONENTS, + ) url = reverse("goods:archive_restore", kwargs={"pk": str(good.id)}) for version_count, is_archived in enumerate([True, False], start=1): @@ -613,7 +618,7 @@ def test_edit_archive_status(self): ] ) def test_only_archive_field_can_be_updated(self, initial, data, expected): - good = GoodFactory(organisation=self.organisation, **initial) + good = GoodFactory(organisation=self.organisation, status=GoodStatus.SUBMITTED, **initial) url = reverse("goods:archive_restore", kwargs={"pk": str(good.id)}) response = self.client.put(url, data, **self.exporter_headers) @@ -624,6 +629,39 @@ def test_only_archive_field_can_be_updated(self, initial, data, expected): actual = {"name": good.name, "is_archived": good.is_archived} self.assertEqual(actual, expected) + @parameterized.expand( + [ + [ + GoodStatus.DRAFT, + {"is_archived": True}, + status.HTTP_404_NOT_FOUND, + {"is_archived": None}, + ], + [ + GoodStatus.SUBMITTED, + {"is_archived": True}, + status.HTTP_200_OK, + {"is_archived": True}, + ], + [ + GoodStatus.VERIFIED, + {"is_archived": True}, + status.HTTP_200_OK, + {"is_archived": True}, + ], + ] + ) + def test_good_with_archivable_status_can_only_be_archived(self, good_status, data, expected_status_code, expected): + good = GoodFactory(organisation=self.organisation, status=good_status) + url = reverse("goods:archive_restore", kwargs={"pk": str(good.id)}) + + response = self.client.put(url, data, **self.exporter_headers) + self.assertEqual(response.status_code, expected_status_code) + + good.refresh_from_db() + actual = {"is_archived": good.is_archived} + self.assertEqual(actual, expected) + class GoodsAttachingTests(DataTestClient): def setUp(self): diff --git a/api/goods/views.py b/api/goods/views.py index 47832eb5ca..d2e2b5607e 100644 --- a/api/goods/views.py +++ b/api/goods/views.py @@ -204,6 +204,7 @@ def get_queryset(self): return Good.objects.filter( organisation=organisation, + status__in=GoodStatus.archivable_statuses(), ) From b8117e3fcf3fddbd4919d9caaf1a01d62831d465 Mon Sep 17 00:00:00 2001 From: Gurdeep Atwal Date: Tue, 2 Jul 2024 19:48:58 +0100 Subject: [PATCH 62/73] include regieme ref. --- api/cases/serializers.py | 1 + api/cases/tests/test_case_search.py | 1 + api/cases/tests/test_get_case.py | 44 +++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+) diff --git a/api/cases/serializers.py b/api/cases/serializers.py index d7c4784502..5fed53b11c 100644 --- a/api/cases/serializers.py +++ b/api/cases/serializers.py @@ -213,6 +213,7 @@ class DenialMatchOnApplicationSummarySerializer(serializers.Serializer): """ name = serializers.CharField(source="denial_entity.name") + regime_reg_ref = serializers.CharField(source="denial_entity.denial.regime_reg_ref") reference = serializers.CharField(source="denial_entity.denial.reference") category = serializers.CharField() address = serializers.CharField(source="denial_entity.address") diff --git a/api/cases/tests/test_case_search.py b/api/cases/tests/test_case_search.py index bafb697640..44ddf9b907 100644 --- a/api/cases/tests/test_case_search.py +++ b/api/cases/tests/test_case_search.py @@ -1093,6 +1093,7 @@ def test_api_success(self): "reference": self.denial_entity.denial.reference, "category": self.denial_on_application.category, "address": self.denial_entity.address, + "regime_reg_ref": self.denial_on_application.denial_entity.denial.regime_reg_ref, } ], ) diff --git a/api/cases/tests/test_get_case.py b/api/cases/tests/test_get_case.py index b704de742e..22167d3672 100644 --- a/api/cases/tests/test_get_case.py +++ b/api/cases/tests/test_get_case.py @@ -3,6 +3,7 @@ from django.contrib.contenttypes.models import ContentType from django.urls import reverse from django.utils import timezone +from api.applications.tests.factories import DenialMatchOnApplicationFactory from api.audit_trail.enums import AuditType from api.audit_trail.models import Audit from api.cases.models import Case, CaseQueue @@ -399,3 +400,46 @@ def test_cases_endpoint_with_sub_status(self): data = response.json() self.assertEqual(data["results"]["cases"][0]["sub_status"]["name"], sub_status.name) + + def test_case_returns_expected_denial_matches(self): + """ + Given a case with a denial match + When the case is retrieved + Then the denial match json data is as expected. + """ + case = self.submit_application(self.standard_application) + denial_match = DenialMatchOnApplicationFactory(application=case) + url = reverse("cases:case", kwargs={"pk": case.id}) + + expected_denial_matches = [ + { + "id": str(denial_match.id), + "application": str(case.id), + "denial_entity": { + "id": str(denial_match.denial_entity.id), + "created_by": str(denial_match.denial_entity.created_by_id), + "name": denial_match.denial_entity.name, + "address": denial_match.denial_entity.address, + "regime_reg_ref": denial_match.denial_entity.denial.regime_reg_ref, + "notifying_government": denial_match.denial_entity.denial.notifying_government, + "country": denial_match.denial_entity.country, + "denial_cle": denial_match.denial_entity.denial.denial_cle, + "item_description": denial_match.denial_entity.denial.item_description, + "end_use": denial_match.denial_entity.denial.end_use, + "is_revoked": denial_match.denial_entity.denial.is_revoked, + "is_revoked_comment": denial_match.denial_entity.denial.is_revoked_comment, + "reason_for_refusal": denial_match.denial_entity.denial.reason_for_refusal, + "entity_type": denial_match.denial_entity.entity_type, + "reference": denial_match.denial_entity.denial.reference, + "denial": str(denial_match.denial_entity.denial.id), + }, + "category": denial_match.category, + } + ] + + response = self.client.get(url, **self.gov_headers) + + response_data = response.json() + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response_data["case"]["data"]["denial_matches"], expected_denial_matches) From 7a0e8ca3b4144628456112bad08541ab7a15474b Mon Sep 17 00:00:00 2001 From: Kevin Carrogan Date: Wed, 3 Jul 2024 17:37:32 +0100 Subject: [PATCH 63/73] Replace status properties view with DRF view --- api/staticdata/statuses/models.py | 4 +++ api/staticdata/statuses/serializers.py | 10 ++++++ api/staticdata/statuses/tests/test_views.py | 34 +++++++++++++++++++++ api/staticdata/statuses/views.py | 20 +++++------- 4 files changed, 55 insertions(+), 13 deletions(-) create mode 100644 api/staticdata/statuses/tests/test_views.py diff --git a/api/staticdata/statuses/models.py b/api/staticdata/statuses/models.py index d6009eb18c..2deaedac2f 100644 --- a/api/staticdata/statuses/models.py +++ b/api/staticdata/statuses/models.py @@ -41,6 +41,10 @@ def is_read_only(self): def is_read_only(cls): return Q(status__in=CaseStatusEnum.read_only_statuses()) + @property + def is_major_editable(self): + return CaseStatusEnum.can_invoke_major_edit(self.status) + def natural_key(self): return (self.status,) diff --git a/api/staticdata/statuses/serializers.py b/api/staticdata/statuses/serializers.py index b74114ab62..a23612c5a4 100644 --- a/api/staticdata/statuses/serializers.py +++ b/api/staticdata/statuses/serializers.py @@ -28,6 +28,16 @@ class Meta: ) +class CaseStatusPropertiesSerializer(serializers.ModelSerializer): + class Meta: + model = CaseStatus + fields = ( + "is_terminal", + "is_read_only", + "is_major_editable", + ) + + class CaseSubStatusSerializer(serializers.ModelSerializer): class Meta: model = CaseSubStatus diff --git a/api/staticdata/statuses/tests/test_views.py b/api/staticdata/statuses/tests/test_views.py new file mode 100644 index 0000000000..d4aba1a24a --- /dev/null +++ b/api/staticdata/statuses/tests/test_views.py @@ -0,0 +1,34 @@ +import json + +from rest_framework import status + +from django.urls import reverse + +from test_helpers.clients import DataTestClient + +from api.staticdata.statuses.factories import CaseStatusFactory + + +class StatusPropertiesTests(DataTestClient): + def test_get_status(self): + case_status = CaseStatusFactory( + status="madeup", + ) + + url = reverse( + "staticdata:statuses:case_status_properties", + kwargs={ + "status": case_status.status, + }, + ) + response = self.client.get(url, **self.exporter_headers) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual( + json.loads(response.content), + { + "is_read_only": case_status.is_read_only, + "is_terminal": case_status.is_terminal, + "is_major_editable": case_status.is_major_editable, + }, + ) diff --git a/api/staticdata/statuses/views.py b/api/staticdata/statuses/views.py index c3584753da..a3f97bd803 100644 --- a/api/staticdata/statuses/views.py +++ b/api/staticdata/statuses/views.py @@ -1,11 +1,13 @@ from django.http import JsonResponse + +from rest_framework.generics import RetrieveAPIView from rest_framework.status import HTTP_200_OK from rest_framework.views import APIView from api.cases.views.search.service import get_case_status_list from api.core.authentication import SharedAuthentication -from api.staticdata.statuses.enums import CaseStatusEnum from api.staticdata.statuses.models import CaseStatus +from api.staticdata.statuses.serializers import CaseStatusPropertiesSerializer class StatusesAsList(APIView): @@ -16,16 +18,8 @@ def get(self, request): return JsonResponse(data={"statuses": statuses}, status=HTTP_200_OK) -class StatusProperties(APIView): +class StatusProperties(RetrieveAPIView): authentication_classes = (SharedAuthentication,) - - def get(self, request, status): - """Return is_read_only and is_terminal properties for a case status.""" - case_status = CaseStatus.objects.get(status=status) - return JsonResponse( - data={ - "is_read_only": CaseStatusEnum.is_read_only(case_status), - "is_terminal": CaseStatusEnum.is_terminal(case_status), - }, - status=HTTP_200_OK, - ) + queryset = CaseStatus.objects.all() + serializer_class = CaseStatusPropertiesSerializer + lookup_field = "status" From 5816ede6e70201f1c5e2a6a249bfd77ed9d0c151 Mon Sep 17 00:00:00 2001 From: Kevin Carrogan Date: Wed, 3 Jul 2024 20:00:09 +0100 Subject: [PATCH 64/73] Simplify method to get all case statuses on enum --- api/staticdata/statuses/enums.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/api/staticdata/statuses/enums.py b/api/staticdata/statuses/enums.py index c057057c59..b11bf52b95 100644 --- a/api/staticdata/statuses/enums.py +++ b/api/staticdata/statuses/enums.py @@ -213,14 +213,7 @@ def as_list(cls): @classmethod def all(cls): - _all = [] - for name, _type in CaseStatusEnum.__dict__.items(): - if not name.isupper(): - continue - if type(_type) is not str: - continue - _all.append(getattr(cls, name)) - return _all + return [getattr(cls, param) for param in dir(cls) if param.isupper()] class CaseStatusIdEnum: From b804170ce3fc36f6effc00056281632dac0bfce1 Mon Sep 17 00:00:00 2001 From: Kevin Carrogan Date: Wed, 3 Jul 2024 20:23:43 +0100 Subject: [PATCH 65/73] Skip deleting key values that don't exist when seeding --- api/staticdata/management/SeedCommand.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/api/staticdata/management/SeedCommand.py b/api/staticdata/management/SeedCommand.py index 47cead3ef8..9c20bcc014 100644 --- a/api/staticdata/management/SeedCommand.py +++ b/api/staticdata/management/SeedCommand.py @@ -83,7 +83,8 @@ def update_or_create(model: models.Model, rows: list, exclude=None): obj = model.objects.filter(id=obj_id) if not obj.exists(): for key in exclude: - del row[key] + if key in row: + del row[key] model.objects.create(**row) if not settings.SUPPRESS_TEST_OUTPUT: print(f"CREATED {model.__name__}: {dict(row)}") @@ -96,7 +97,8 @@ def update_if_not_equal(obj: QuerySet, row: dict, exclude=None): # `delete_unused_objects` exclude = exclude or [] for key in exclude: - del row[key] + if key in row: + del row[key] attributes = {k: v for k, v in row.items() if k != "id"} obj = obj.exclude(**attributes) if obj.exists(): From 9e93c1087c4cf5e30be25a1ca854267aba08cc8b Mon Sep 17 00:00:00 2001 From: Kevin Carrogan Date: Wed, 3 Jul 2024 21:08:35 +0100 Subject: [PATCH 66/73] Add missing init files --- api/external_data/tests/__init__.py | 0 api/staticdata/statuses/tests/__init__.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 api/external_data/tests/__init__.py create mode 100644 api/staticdata/statuses/tests/__init__.py diff --git a/api/external_data/tests/__init__.py b/api/external_data/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/staticdata/statuses/tests/__init__.py b/api/staticdata/statuses/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From 06253e7af0417e351517bf06e46cbc36322538cf Mon Sep 17 00:00:00 2001 From: Kevin Carrogan Date: Wed, 3 Jul 2024 21:10:12 +0100 Subject: [PATCH 67/73] Split out decorator for editable and major editable --- api/applications/views/applications.py | 9 +-- api/applications/views/documents.py | 9 +-- api/applications/views/end_use_details.py | 7 +- api/applications/views/external_locations.py | 7 +- api/applications/views/goods.py | 8 +-- api/applications/views/parties.py | 4 +- .../views/temporary_export_details.py | 4 +- api/core/decorators.py | 64 +++++++++---------- api/core/tests/test_decorators.py | 15 +++-- api/staticdata/statuses/enums.py | 8 +++ 10 files changed, 78 insertions(+), 57 deletions(-) diff --git a/api/applications/views/applications.py b/api/applications/views/applications.py index 369454b1b3..464a7f5027 100644 --- a/api/applications/views/applications.py +++ b/api/applications/views/applications.py @@ -70,7 +70,8 @@ from api.core.authentication import ExporterAuthentication, SharedAuthentication, GovAuthentication from api.core.constants import ExporterPermissions, GovPermissions, AutoGeneratedDocuments from api.core.decorators import ( - application_in_state, + application_is_editable, + application_is_major_editable, authorised_to_view_application, allowed_application_types, ) @@ -226,7 +227,7 @@ def get(self, request, pk): return JsonResponse(data=data, status=status.HTTP_200_OK) @authorised_to_view_application(ExporterUser) - @application_in_state(is_editable=True) + @application_is_editable def put(self, request, pk): """ Update an application instance @@ -299,7 +300,7 @@ class ApplicationSubmission(APIView): authentication_classes = (ExporterAuthentication,) @transaction.atomic - @application_in_state(is_major_editable=True) + @application_is_major_editable @authorised_to_view_application(ExporterUser) def put(self, request, pk): """ @@ -849,7 +850,7 @@ class ApplicationRouteOfGoods(UpdateAPIView): authentication_classes = (ExporterAuthentication,) @authorised_to_view_application(ExporterUser) - @application_in_state(is_major_editable=True) + @application_is_major_editable @allowed_application_types([CaseTypeSubTypeEnum.OPEN, CaseTypeSubTypeEnum.STANDARD]) def put(self, request, pk): """Update an application instance with route of goods data.""" diff --git a/api/applications/views/documents.py b/api/applications/views/documents.py index e392473c22..ba74ba9f4d 100644 --- a/api/applications/views/documents.py +++ b/api/applications/views/documents.py @@ -12,7 +12,8 @@ from api.core.decorators import ( authorised_to_view_application, allowed_application_types, - application_in_state, + application_is_editable, + application_is_major_editable, ) from api.goodstype.helpers import get_goods_type from api.users.models import ExporterUser @@ -37,7 +38,7 @@ def get(self, request, pk): @transaction.atomic @authorised_to_view_application(ExporterUser) - @application_in_state(is_editable=True) + @application_is_editable def post(self, request, pk): """ Upload additional document onto an application @@ -62,7 +63,7 @@ def get(self, request, pk, doc_pk): @transaction.atomic @authorised_to_view_application(ExporterUser) - @application_in_state(is_editable=True) + @application_is_editable def delete(self, request, pk, doc_pk): """ Delete an additional document on an application @@ -86,7 +87,7 @@ def get(self, request, pk, goods_type_pk): @transaction.atomic @allowed_application_types([CaseTypeSubTypeEnum.HMRC]) - @application_in_state(is_major_editable=True) + @application_is_major_editable @authorised_to_view_application(ExporterUser) def post(self, request, pk, goods_type_pk): goods_type = get_goods_type(goods_type_pk) diff --git a/api/applications/views/end_use_details.py b/api/applications/views/end_use_details.py index c9a40ad79f..3eea09a8bb 100644 --- a/api/applications/views/end_use_details.py +++ b/api/applications/views/end_use_details.py @@ -6,7 +6,10 @@ from api.applications.libraries.edit_applications import save_and_audit_end_use_details from api.applications.libraries.get_applications import get_application from api.core.authentication import ExporterAuthentication -from api.core.decorators import authorised_to_view_application, application_in_state +from api.core.decorators import ( + authorised_to_view_application, + application_is_major_editable, +) from api.users.models import ExporterUser @@ -14,7 +17,7 @@ class EndUseDetails(UpdateAPIView): authentication_classes = (ExporterAuthentication,) @authorised_to_view_application(ExporterUser) - @application_in_state(is_major_editable=True) + @application_is_major_editable def put(self, request, pk): application = get_application(pk) serializer = get_application_end_use_details_update_serializer(application) diff --git a/api/applications/views/external_locations.py b/api/applications/views/external_locations.py index 7a80b490eb..2ffd72cae3 100644 --- a/api/applications/views/external_locations.py +++ b/api/applications/views/external_locations.py @@ -12,7 +12,10 @@ from api.audit_trail.enums import AuditType from api.cases.enums import CaseTypeEnum from api.core.authentication import ExporterAuthentication -from api.core.decorators import authorised_to_view_application, application_in_state +from api.core.decorators import ( + authorised_to_view_application, + application_is_editable, +) from lite_content.lite_api.strings import ExternalLocations from api.organisations.enums import LocationType from api.organisations.libraries.get_external_location import get_location @@ -44,7 +47,7 @@ def get(self, request, pk): @transaction.atomic @authorised_to_view_application(ExporterUser) - @application_in_state(is_editable=True) + @application_is_editable def post(self, request, pk): application = get_application(pk) data = request.data diff --git a/api/applications/views/goods.py b/api/applications/views/goods.py index 806a999f3c..936cc519da 100644 --- a/api/applications/views/goods.py +++ b/api/applications/views/goods.py @@ -23,7 +23,7 @@ from api.core.decorators import ( authorised_to_view_application, allowed_application_types, - application_in_state, + application_is_major_editable, ) from api.core.exceptions import BadRequestError from api.flags.enums import SystemFlags @@ -67,7 +67,7 @@ def get(self, request, pk): CaseTypeSubTypeEnum.F680, ] ) - @application_in_state(is_major_editable=True) + @application_is_major_editable @authorised_to_view_application(ExporterUser) def post(self, request, pk): data = request.data @@ -271,7 +271,7 @@ def get(self, request, pk, good_pk): return JsonResponse({"documents": serializer.data}, status=status.HTTP_200_OK) - @application_in_state(is_major_editable=True) + @application_is_major_editable @authorised_to_view_application(ExporterUser) def post(self, request, pk, good_pk): data = request.data @@ -300,7 +300,7 @@ def post(self, request, pk, good_pk): delete_uploaded_document(data) return JsonResponse({"errors": serializer.errors}, status=status.HTTP_400_BAD_REQUEST) - @application_in_state(is_major_editable=True) + @application_is_major_editable @authorised_to_view_application(ExporterUser) def delete(self, request, pk, good_pk): delete_uploaded_document(request.data) diff --git a/api/applications/views/parties.py b/api/applications/views/parties.py index 1bfaec4afe..86d9c8b9fc 100644 --- a/api/applications/views/parties.py +++ b/api/applications/views/parties.py @@ -10,7 +10,7 @@ from api.core.decorators import ( authorised_to_view_application, allowed_party_type_for_open_application_goodstype_category, - application_in_state, + application_is_major_editable, ) from api.core.helpers import str_to_bool from lite_content.lite_api import strings @@ -37,7 +37,7 @@ def party(self): @allowed_party_type_for_open_application_goodstype_category() @authorised_to_view_application(ExporterUser) - @application_in_state(is_major_editable=True) + @application_is_major_editable def post(self, request, pk): """ Add a party to an application diff --git a/api/applications/views/temporary_export_details.py b/api/applications/views/temporary_export_details.py index 84c9da4186..2d2172085d 100644 --- a/api/applications/views/temporary_export_details.py +++ b/api/applications/views/temporary_export_details.py @@ -12,7 +12,7 @@ from api.core.decorators import ( authorised_to_view_application, allowed_application_types, - application_in_state, + application_is_major_editable, ) from api.users.models import ExporterUser @@ -22,7 +22,7 @@ class TemporaryExportDetails(UpdateAPIView): @authorised_to_view_application(ExporterUser) @allowed_application_types([CaseTypeSubTypeEnum.OPEN, CaseTypeSubTypeEnum.STANDARD]) - @application_in_state(is_major_editable=True) + @application_is_major_editable def put(self, request, pk): application = get_application(pk) if application.export_type == ApplicationExportType.PERMANENT: diff --git a/api/core/decorators.py b/api/core/decorators.py index 473fcabe72..da52333f61 100644 --- a/api/core/decorators.py +++ b/api/core/decorators.py @@ -67,45 +67,45 @@ def inner(request, *args, **kwargs): return decorator -def application_in_state(is_editable: bool = False, is_major_editable: bool = False) -> Callable: - """ - Checks if application is in an editable or major-editable state - """ - - def decorator(func): - @wraps(func) +def application_in_status(status_check_func): + @wraps(status_check_func) + def decorator(view_func): + @wraps(view_func) def inner(request, *args, **kwargs): application_status = _get_application(request, kwargs).values_list("status__status", flat=True)[0] + has_status, error = status_check_func(application_status) + if has_status: + return view_func(request, *args, **kwargs) + return JsonResponse( + data={"errors": {"non_field_errors": [error]}}, + status=status.HTTP_400_BAD_REQUEST, + ) - if is_editable and application_status in CaseStatusEnum.read_only_statuses(): - return JsonResponse( - data={ - "errors": { - "non_field_errors": [ - strings.Applications.Generic.INVALID_OPERATION_FOR_READ_ONLY_CASE_ERROR - ] - } - }, - status=status.HTTP_400_BAD_REQUEST, - ) + return inner - if is_major_editable and application_status not in CaseStatusEnum.major_editable_statuses(): - return JsonResponse( - data={ - "errors": { - "non_field_errors": [ - strings.Applications.Generic.INVALID_OPERATION_FOR_NON_DRAFT_OR_MAJOR_EDIT_CASE_ERROR - ] - } - }, - status=status.HTTP_400_BAD_REQUEST, - ) + return decorator - return func(request, *args, **kwargs) - return inner +@application_in_status +def application_is_editable(application_status): + """ + Checks if application is editable + """ + return ( + CaseStatusEnum.is_editable(application_status), + strings.Applications.Generic.INVALID_OPERATION_FOR_READ_ONLY_CASE_ERROR, + ) - return decorator + +@application_in_status +def application_is_major_editable(application_status): + """ + Checks if application is major editable + """ + return ( + CaseStatusEnum.is_major_editable_status(application_status), + strings.Applications.Generic.INVALID_OPERATION_FOR_NON_DRAFT_OR_MAJOR_EDIT_CASE_ERROR, + ) def authorised_to_view_application(user_type: Union[Type[GovUser], Type[ExporterUser]]) -> Callable: diff --git a/api/core/tests/test_decorators.py b/api/core/tests/test_decorators.py index 6dfe481b2d..1b75c1502a 100644 --- a/api/core/tests/test_decorators.py +++ b/api/core/tests/test_decorators.py @@ -7,7 +7,12 @@ from api.applications.libraries.case_status_helpers import get_case_statuses from api.cases.enums import CaseTypeSubTypeEnum from api.core.authentication import ORGANISATION_ID -from api.core.decorators import allowed_application_types, application_in_state, authorised_to_view_application +from api.core.decorators import ( + allowed_application_types, + application_is_editable, + application_is_major_editable, + authorised_to_view_application, +) from lite_content.lite_api import strings from api.organisations.tests.factories import OrganisationFactory from api.staticdata.statuses.enums import CaseStatusEnum @@ -52,7 +57,7 @@ def test_application_in_state_editable_success(self, editable_status): application.status = get_case_status_by_status(editable_status) application.save() - @application_in_state(is_editable=True) + @application_is_editable def a_view(request, *args, **kwargs): return HttpResponse() @@ -65,7 +70,7 @@ def test_application_in_state_editable_failure(self): application.status = CaseStatus.objects.get(status=application_status) application.save() - @application_in_state(is_editable=True) + @application_is_editable def a_view(request, *args, **kwargs): return HttpResponse() @@ -80,7 +85,7 @@ def test_application_in_state_major_editable_success(self): application.status = CaseStatus.objects.get(status=CaseStatusEnum.major_editable_statuses()[0]) application.save() - @application_in_state(is_major_editable=True) + @application_is_major_editable def a_view(request, *args, **kwargs): return HttpResponse() @@ -92,7 +97,7 @@ def test_application_in_state_major_editable_failure(self): application.status = CaseStatus.objects.get(status=CaseStatusEnum.read_only_statuses()[0]) application.save() - @application_in_state(is_major_editable=True) + @application_is_major_editable def a_view(request, *args, **kwargs): return HttpResponse() diff --git a/api/staticdata/statuses/enums.py b/api/staticdata/statuses/enums.py index b11bf52b95..f765b035b1 100644 --- a/api/staticdata/statuses/enums.py +++ b/api/staticdata/statuses/enums.py @@ -174,6 +174,10 @@ def get_value(cls, status): def is_read_only(cls, status): return status in cls.read_only_statuses() + @classmethod + def is_editable(cls, status): + return not cls.is_read_only(status) + @classmethod def is_terminal(cls, status): return status in cls._terminal_statuses @@ -190,6 +194,10 @@ def read_only_statuses(cls): def major_editable_statuses(cls): return cls._major_editable_statuses + @classmethod + def is_major_editable_status(cls, status): + return status in cls._major_editable_statuses + @classmethod def can_invoke_major_edit_statuses(cls): return cls._can_invoke_major_edit_statuses From d0424accc06bd23d6596b6b9f5a6b4c564969cf9 Mon Sep 17 00:00:00 2001 From: Kevin Carrogan Date: Wed, 3 Jul 2024 21:19:36 +0100 Subject: [PATCH 68/73] Make a distinction between major editable and major edit invokable status --- api/staticdata/statuses/models.py | 4 ++++ api/staticdata/statuses/serializers.py | 1 + api/staticdata/statuses/tests/test_views.py | 1 + 3 files changed, 6 insertions(+) diff --git a/api/staticdata/statuses/models.py b/api/staticdata/statuses/models.py index 2deaedac2f..c6a950274d 100644 --- a/api/staticdata/statuses/models.py +++ b/api/staticdata/statuses/models.py @@ -43,6 +43,10 @@ def is_read_only(cls): @property def is_major_editable(self): + return CaseStatusEnum.is_major_editable_status(self.status) + + @property + def can_invoke_major_editable(self): return CaseStatusEnum.can_invoke_major_edit(self.status) def natural_key(self): diff --git a/api/staticdata/statuses/serializers.py b/api/staticdata/statuses/serializers.py index a23612c5a4..8d854bd163 100644 --- a/api/staticdata/statuses/serializers.py +++ b/api/staticdata/statuses/serializers.py @@ -35,6 +35,7 @@ class Meta: "is_terminal", "is_read_only", "is_major_editable", + "can_invoke_major_editable", ) diff --git a/api/staticdata/statuses/tests/test_views.py b/api/staticdata/statuses/tests/test_views.py index d4aba1a24a..9da28cf79b 100644 --- a/api/staticdata/statuses/tests/test_views.py +++ b/api/staticdata/statuses/tests/test_views.py @@ -30,5 +30,6 @@ def test_get_status(self): "is_read_only": case_status.is_read_only, "is_terminal": case_status.is_terminal, "is_major_editable": case_status.is_major_editable, + "can_invoke_major_editable": case_status.can_invoke_major_editable, }, ) From 3a7e8891dcfa1fd23d7821ddd2d7220761d34d0d Mon Sep 17 00:00:00 2001 From: Kevin Carrogan Date: Fri, 5 Jul 2024 12:11:00 +0100 Subject: [PATCH 69/73] Remove checking out master on temporary lite-frontend branch --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index aa3372b93d..8a29ab99d2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -531,7 +531,6 @@ jobs: TEMPORARY_BRANCH_NAME+=$(cat /proc/sys/kernel/random/uuid) git clone git@github.com:uktrade/lite-frontend.git cd lite-frontend - git checkout master git config user.email $GIT_EMAIL git config user.name "LITE CI" git checkout -b $TEMPORARY_BRANCH_NAME From 2a4f577494e0757a22d198718858048a1c3cface Mon Sep 17 00:00:00 2001 From: Arun Siluvery Date: Fri, 5 Jul 2024 15:24:03 +0100 Subject: [PATCH 70/73] Fix failing test to use updated archive url --- api/applications/tests/test_removing_goods.py | 1 - api/applications/views/tests/test_parties.py | 1 - api/goods/tests/test_views.py | 4 ++-- test_helpers/clients.py | 2 +- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/api/applications/tests/test_removing_goods.py b/api/applications/tests/test_removing_goods.py index 49d561507e..912c5845c1 100644 --- a/api/applications/tests/test_removing_goods.py +++ b/api/applications/tests/test_removing_goods.py @@ -11,7 +11,6 @@ from api.goods.models import Good, FirearmGoodDetails from api.users.models import UserOrganisationRelationship from api.staticdata.statuses.enums import CaseStatusEnum -from api.staticdata.statuses.libraries.get_case_status import get_case_status_by_status from api.staticdata.units.enums import Units from test_helpers.clients import DataTestClient from api.users.libraries.user_to_token import user_to_token diff --git a/api/applications/views/tests/test_parties.py b/api/applications/views/tests/test_parties.py index e6b98f36b8..d1b20f3e8f 100644 --- a/api/applications/views/tests/test_parties.py +++ b/api/applications/views/tests/test_parties.py @@ -1,5 +1,4 @@ from django.urls import reverse -from parameterized import parameterized from rest_framework import status from api.applications.tests.factories import ( diff --git a/api/goods/tests/test_views.py b/api/goods/tests/test_views.py index bdf70320ae..1cee7173a4 100644 --- a/api/goods/tests/test_views.py +++ b/api/goods/tests/test_views.py @@ -178,8 +178,8 @@ def test_goods_list_excludes_archived_goods(self): # archive one good good = all_goods[-1] - edit_url = reverse("goods:good", kwargs={"pk": str(good.id)}) - response = self.client.put(edit_url, {"is_archived": True}, **self.exporter_headers) + archive_restore_url = reverse("goods:archive_restore", kwargs={"pk": str(good.id)}) + response = self.client.put(archive_restore_url, {"is_archived": True}, **self.exporter_headers) self.assertEqual(response.status_code, status.HTTP_200_OK) # check archive good is excluded diff --git a/test_helpers/clients.py b/test_helpers/clients.py index 8cc134a58e..668724eba8 100644 --- a/test_helpers/clients.py +++ b/test_helpers/clients.py @@ -3,7 +3,7 @@ import uuid import sys from django.utils import timezone -from typing import List, Tuple +from typing import Tuple import django.utils.timezone from django.db import connection From d0cfe7d90b191bc0881bc2d5353adf2e6182e1d4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Jul 2024 13:13:42 +0000 Subject: [PATCH 71/73] Bump certifi from 2024.6.2 to 2024.7.4 Bumps [certifi](https://github.com/certifi/python-certifi) from 2024.6.2 to 2024.7.4. - [Commits](https://github.com/certifi/python-certifi/compare/2024.06.02...2024.07.04) --- updated-dependencies: - dependency-name: certifi dependency-type: indirect ... Signed-off-by: dependabot[bot] --- Pipfile.lock | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 317598a205..8edd5ee78c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -116,11 +116,12 @@ }, "certifi": { "hashes": [ - "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516", - "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56" + "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b", + "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90" ], + "index": "pypi", "markers": "python_version >= '3.6'", - "version": "==2024.6.2" + "version": "==2024.7.4" }, "cffi": { "hashes": [ From 9a6d01fb401842a6db2ea609c00fd5da55089660 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Jul 2024 14:11:00 +0000 Subject: [PATCH 72/73] Bump djangorestframework from 3.14.0 to 3.15.2 Bumps [djangorestframework](https://github.com/encode/django-rest-framework) from 3.14.0 to 3.15.2. - [Release notes](https://github.com/encode/django-rest-framework/releases) - [Commits](https://github.com/encode/django-rest-framework/compare/3.14.0...3.15.2) --- updated-dependencies: - dependency-name: djangorestframework dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- Pipfile | 2 +- Pipfile.lock | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Pipfile b/Pipfile index 9e198d4261..5a5e27ee41 100644 --- a/Pipfile +++ b/Pipfile @@ -36,7 +36,7 @@ django-health-check = "~=3.18.1" django-model-utils = "~=4.3.1" django-sortedm2m = "~=3.1.1" django-staff-sso-client = "~=4.2.1" -djangorestframework = "~=3.14.0" +djangorestframework = "~=3.15.2" elasticsearch = "<7.14.0" markdown = "~=3.4.1" mohawk = "~=1.1.0" diff --git a/Pipfile.lock b/Pipfile.lock index 8edd5ee78c..27cf2db737 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "c6c946c69f5bee64c779b1c82f29cecbfe5c23d1c819e5e041d387e221a66086" + "sha256": "4f98561bdf3763491c38fdf9b46517823a54548fd1eb84ef79131de86c27b9b2" }, "pipfile-spec": 6, "requires": { @@ -610,12 +610,12 @@ }, "djangorestframework": { "hashes": [ - "sha256:579a333e6256b09489cbe0a067e66abe55c6595d8926be6b99423786334350c8", - "sha256:eb63f58c9f218e1a7d064d17a70751f528ed4e1d35547fdade9aaf4cd103fd08" + "sha256:2b8871b062ba1aefc2de01f773875441a961fefbf79f5eed1e32b2f096944b20", + "sha256:36fe88cd2d6c6bec23dca9804bab2ba5517a8bb9d8f47ebc68981b56840107ad" ], "index": "pypi", - "markers": "python_version >= '3.6'", - "version": "==3.14.0" + "markers": "python_version >= '3.8'", + "version": "==3.15.2" }, "docopt": { "hashes": [ From 1d1505cccec85cba55881753fb529c31c1fc2863 Mon Sep 17 00:00:00 2001 From: Arun Siluvery Date: Wed, 3 Jul 2024 15:09:53 +0100 Subject: [PATCH 73/73] Add factory to create a draft application The idea is to replace create_draft_standard_application() with this factory as that is heavily overloaded with many things. --- api/applications/tests/factories.py | 30 ++++++++++++++++++- .../tests/test_view_application.py | 3 +- api/cases/tests/test_good_precedents_view.py | 2 +- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/api/applications/tests/factories.py b/api/applications/tests/factories.py index d06a3e798d..f0e0e7fc0a 100644 --- a/api/applications/tests/factories.py +++ b/api/applications/tests/factories.py @@ -20,7 +20,7 @@ from api.staticdata.statuses.models import CaseStatus from api.goods.tests.factories import GoodFactory from api.organisations.tests.factories import OrganisationFactory, SiteFactory, ExternalLocationFactory -from api.parties.tests.factories import PartyFactory +from api.parties.tests.factories import ConsigneeFactory, EndUserFactory, PartyFactory, ThirdPartyFactory from api.users.tests.factories import ExporterUserFactory, GovUserFactory from api.staticdata.control_list_entries.helpers import get_control_list_entry from api.staticdata.regimes.helpers import get_regime_entry @@ -49,6 +49,8 @@ class StandardApplicationFactory(factory.django.DjangoModelFactory): is_shipped_waybill_or_lading = True non_waybill_or_lading_route_details = None is_mod_security_approved = False + goods_starting_point = StandardApplication.GB + goods_recipients = StandardApplication.DIRECT_TO_END_USER submitted_by = factory.SubFactory(ExporterUserFactory) class Meta: @@ -210,3 +212,29 @@ class SanctionMatchFactory(factory.django.DjangoModelFactory): class Meta: model = SanctionMatch + + +class DraftStandardApplicationFactory(StandardApplicationFactory): + goods_starting_point = StandardApplication.GB + goods_recipients = StandardApplication.VIA_CONSIGNEE + + @classmethod + def _create(cls, model_class, *args, **kwargs): + obj = model_class(*args, **kwargs) + obj.status = get_case_status_by_status(CaseStatusEnum.DRAFT) + obj.save() + + GoodOnApplicationFactory(application=obj, good=GoodFactory(organisation=obj.organisation)) + + PartyOnApplicationFactory(application=obj, party=EndUserFactory(organisation=obj.organisation)) + + if kwargs["goods_recipients"] in [ + StandardApplication.VIA_CONSIGNEE, + StandardApplication.VIA_CONSIGNEE_AND_THIRD_PARTIES, + ]: + PartyOnApplicationFactory(application=obj, party=ConsigneeFactory(organisation=obj.organisation)) + + if kwargs["goods_recipients"] == StandardApplication.VIA_CONSIGNEE_AND_THIRD_PARTIES: + PartyOnApplicationFactory(application=obj, party=ThirdPartyFactory(organisation=obj.organisation)) + + return obj diff --git a/api/applications/tests/test_view_application.py b/api/applications/tests/test_view_application.py index 75b7b74f04..2dc44f0dc3 100644 --- a/api/applications/tests/test_view_application.py +++ b/api/applications/tests/test_view_application.py @@ -12,6 +12,7 @@ from rest_framework import status from api.applications.models import GoodOnApplication, SiteOnApplication +from api.applications.tests.factories import DraftStandardApplicationFactory from api.cases.enums import AdviceType, CaseTypeEnum from api.organisations.tests.factories import SiteFactory from api.staticdata.statuses.enums import CaseStatusEnum @@ -31,7 +32,7 @@ def test_view_draft_standard_application_list_as_exporter_success(self): Ensure we can get a list of drafts. """ self.exporter_user.set_role(self.organisation, self.exporter_super_user_role) - standard_application = self.create_draft_standard_application(self.organisation) + standard_application = DraftStandardApplicationFactory(organisation=self.organisation) response = self.client.get(self.url, **self.exporter_headers) response_data = response.json()["results"] diff --git a/api/cases/tests/test_good_precedents_view.py b/api/cases/tests/test_good_precedents_view.py index 40eb642209..d974cf5c86 100644 --- a/api/cases/tests/test_good_precedents_view.py +++ b/api/cases/tests/test_good_precedents_view.py @@ -138,7 +138,7 @@ def test_get_with_matching_precedents(self): "%Y-%m-%dT%H:%M:%S.%f" )[:-3] + "Z", - "goods_starting_point": "", + "goods_starting_point": "GB", "regime_entries": [ { "name": "Wassenaar Arrangement",