Add end to end test to verify SPIRE usage data reply

mail/libraries/data_converters.py

@@ -52,16 +52,21 @@ def convert_data_for_licence_data_reply(dto: EmailMessageDto) -> dict:
 
 
 def convert_data_for_usage_data(dto: EmailMessageDto) -> dict:
+    logging.debug("Converting %s", dto)
     data = {
         "usage_data": {},
         "edi_filename": process_attachment(dto.attachment)[0],
         "edi_data": process_attachment(dto.attachment)[1],
     }
-    data["usage_data"]["spire_run_number"] = (
-        new_spire_run_number(int(dto.run_number)) if convert_sender_to_source(dto.sender) in VALID_SENDERS else None
-    )
+    run_number = None
+    source = convert_sender_to_source(dto.sender)
+    logging.debug("Source for %s is %s", dto.sender, source)
+    if source in VALID_SENDERS:
+        run_number = new_spire_run_number(int(dto.run_number))
+    data["usage_data"]["spire_run_number"] = run_number
     data["usage_data"]["hmrc_run_number"] = dto.run_number
     data["usage_data"]["licence_ids"] = get_licence_ids(data["edi_data"])
+    logging.debug("Converted to usage data %s", data)
     return data
 
 

mail/libraries/data_processors.py

@@ -43,6 +43,8 @@ def serialize_email_message(dto: EmailMessageDto) -> Mail or None:
     serializer_class = get_serializer_for_dto(extract_type)
     serializer = serializer_class(instance=instance, data=data, partial=partial)
 
+    logging.debug("About to serialize using %s", serializer_class)
+
     if not serializer.is_valid():
         logging.error("Failed to serialize email (subject: %s) -> %s", dto.subject, serializer.errors)
         raise ValidationError(serializer.errors)
@@ -63,6 +65,7 @@ def convert_dto_data_for_serialization(dto: EmailMessageDto, extract_type) -> di
     :return: new dto for different extract type; corresponding Serializer;
             and existing mail if extract type is of reply. Both serializer and mail could be None
     """
+    logging.debug("Converting data for %s", extract_type)
     if extract_type == ExtractTypeEnum.LICENCE_DATA:
         data = convert_data_for_licence_data(dto)
     elif extract_type == ExtractTypeEnum.LICENCE_REPLY:

mail/tests/files/end_to_end/ILBDOTI_live_CHIEF_usageData_1_202001010000

@@ -0,0 +1,7 @@
+1\fileHeader\CHIEF\SPIRE\usageData\202001010000\1\
+2\licenceUsage\LU12345/00001\insert\SIEL12345\O\
+3\line\1\0\0\
+4\usage\O\112345\R\20200101\0\0\\000111\\\\
+5\end\line\3
+6\end\licenceUsage\5
+7\fileTrailer\1

mail/tests/test_end_to_end.py

@@ -13,7 +13,7 @@
 from mail.celery_tasks import manage_inbox, send_licence_details_to_hmrc
 from mail.enums import ExtractTypeEnum, ReceptionStatusEnum, SourceEnum
 from mail.libraries.helpers import read_file
-from mail.models import LicenceData, LicencePayload, Mail
+from mail.models import LicenceData, LicencePayload, Mail, UsageData
 from mail.servers import MailServer
 
 pytestmark = pytest.mark.django_db
@@ -37,6 +37,7 @@ def set_settings(settings, outgoing_email_user):
     settings.OUTGOING_EMAIL_USER = outgoing_email_user
 
     settings.HMRC_TO_DIT_REPLY_ADDRESS = "[email protected]"
+    settings.HMRC_ADDRESS = "[email protected]"
 
 
 @pytest.fixture()
@@ -86,6 +87,16 @@ def licence_reply_file_body(licence_reply_file_name):
     return read_file(f"mail/tests/files/end_to_end/{licence_reply_file_name}", mode="rb")
 
 
+@pytest.fixture()
+def usage_data_file_name():
+    return "ILBDOTI_live_CHIEF_usageData_1_202001010000"
+
+
+@pytest.fixture()
+def usage_data_file_body(usage_data_file_name):
+    return read_file(f"mail/tests/files/end_to_end/{usage_data_file_name}", mode="rb")
+
+
 @pytest.fixture(autouse=True)
 def hmrc_to_dit_mailserver(mocker):
     auth = BasicAuthentication(
@@ -369,3 +380,56 @@ def test_receive_spire_licence_reply_from_hmrc_e2e(
 
     body = get_smtp_body()
     assert normalise_line_endings(body) == normalise_line_endings(licence_reply_file_body.decode("ascii"))
+
+
+def test_receive_spire_licence_reply_from_hmrc_e2e(
+    usage_data_file_name,
+    usage_data_file_body,
+    get_smtp_message_count,
+    get_smtp_message,
+    get_smtp_body,
+):
+    assert not Mail.objects.exists()
+    assert not UsageData.objects.exists()
+
+    response = requests.post(
+        "http://hmrc-to-dit-mailserver:8025/api/v1/send",
+        json={
+            "From": {"Email": settings.HMRC_TO_DIT_REPLY_ADDRESS, "Name": "HMRC"},
+            "Subject": usage_data_file_name,
+            "To": [{"Email": "[email protected]", "Name": "LITE"}],  # /PS-IGNORE
+            "Attachments": [
+                {
+                    "Content": b64encode(usage_data_file_body).decode("ascii"),
+                    "Filename": usage_data_file_name,
+                }
+            ],
+        },
+    )
+    assert response.status_code == status.HTTP_200_OK
+
+    manage_inbox.delay()
+
+    assert Mail.objects.count() == 1
+    mail = Mail.objects.get()
+    assert mail.extract_type == ExtractTypeEnum.USAGE_DATA
+    assert mail.status == ReceptionStatusEnum.REPLY_SENT
+    assert mail.edi_filename == usage_data_file_name
+    assert normalise_line_endings(mail.edi_data) == normalise_line_endings(usage_data_file_body.decode("ascii"))
+
+    assert UsageData.objects.count() == 1
+    usage_data = UsageData.objects.get()
+    assert usage_data.mail == mail
+    assert usage_data.spire_run_number == 1
+    assert usage_data.hmrc_run_number == 1
+    assert not usage_data.has_lite_data
+    assert usage_data.has_spire_data
+
+    assert get_smtp_message_count() == 1
+
+    _, message = get_smtp_message()
+    assert message["To"] == [{"Name": "", "Address": "[email protected]"}]
+    assert message["Subject"] == usage_data_file_name
+
+    body = get_smtp_body()
+    assert normalise_line_endings(body) == normalise_line_endings(usage_data_file_body.decode("ascii"))