Merge pull request #538 from raft-tech/release/v3.5.2-Sprint-104-update

Release/v3.5.2 sprint 104 update

README.md

@@ -15,7 +15,7 @@ Our vision is to build a new, secure, web-based data reporting system to improve
 |**Frontend Coverage**| [![Codecov-Frontend-Dev](https://codecov.io/gh/raft-tech/TANF-app/branch/develop/graph/badge.svg?flag=dev-frontend)](https://codecov.io/gh/raft-tech/TANF-app?flag=dev-frontend) | [![Codeco-Frontend-HHS](https://codecov.io/gh/HHS/TANF-app/branch/main/graph/badge.svg?flag=main-frontend)](https://codecov.io/gh/HHS/TANF-app?flag=main-frontend)   | [![Codeco-Frontend-HHS](https://codecov.io/gh/HHS/TANF-app/branch/master/graph/badge.svg?flag=master-frontend)](https://codecov.io/gh/HHS/TANF-app?flag=master-frontend) 
 |**Backend Coverage**|  [![Codecov-Backend-Dev](https://codecov.io/gh/raft-tech/TANF-app/branch/develop/graph/badge.svg?flag=dev-backend)](https://codecov.io/gh/raft-tech/TANF-app/branch/develop?flag=dev-backend)|   [![Codecov-Backend-HHS]( https://codecov.io/gh/HHS/TANF-app/branch/main/graph/badge.svg?flag=main-backend)](https://codecov.io/gh/HHS/TANF-app/branch/main?flag=main-backend) |  [![Codecov-Backend-HHS]( https://codecov.io/gh/HHS/TANF-app/branch/master/graph/badge.svg?flag=master-backend)](https://codecov.io/gh/HHS/TANF-app/branch/master?flag=master-backend)
 
-[Link to Current Development Deployments](https://github.com/raft-tech/TANF-app/blob/feat/1860/docs/Technical-Documentation/TDP-environments-README.md#development)
+[Link to Current Development Deployments](./docs/Technical-Documentation/TDP-environments-README.md)
 
 *Due to limitations imposed by Github and occasional slow server response times, some badges may require a page refresh to load.*
 

docs/Technical-Documentation/TDP-environments-README.md

@@ -6,8 +6,7 @@
 | -------- | -------- | -------- | -------- | -------- |
 | A11y | https://tdp-frontend-a11y.app.cloud.gov | https://tdp-frontend-a11y.app.cloud.gov/admin/ |  | Space for accessibility testing |
 | QASP | https://tdp-frontend-qasp.app.cloud.gov | https://tdp-frontend-qasp.app.cloud.gov/admin/ |  | Space for QASP review |
-| raft | https://tdp-frontend-raft.app.cloud.gov | https://tdp-frontend-raft.app.cloud.gov/admin/ | 
- | Space for raft review |
+| raft | https://tdp-frontend-raft.app.cloud.gov | https://tdp-frontend-raft.app.cloud.gov/admin/ |  | Space for raft review |
 
 ![badge](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/andrew-jameson/ded3a260ed8245a5b231ba726b3039df/raw/Live-Environments-raft.json)
 

scripts/deploy-backend.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 ##############################
-# Global Variable Decls 
+# Global Variable Decls
 ##############################
 
 # The deployment strategy you wish to employ ( rolling update or setting up a new environment)
@@ -77,15 +77,15 @@ set_cf_envs()
     else
       cf_cmd="cf set-env $CGAPPNAME_BACKEND $var_name ${!var_name}"
     fi
-    
+
     echo "Setting var : $var_name"
     $cf_cmd
   done
 
 }
 
 # Helper method to generate JWT cert and keys for new environment
-generate_jwt_cert() 
+generate_jwt_cert()
 {
     echo "regenerating JWT cert/key"
     yes 'XX' | openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -nodes -sha256
@@ -94,7 +94,7 @@ generate_jwt_cert()
 }
 
 update_kibana()
-{  
+{
   # Add network policy allowing Kibana to talk to the proxy and to allow the backend to talk to Kibana
   cf add-network-policy "$CGAPPNAME_BACKEND" "$CGAPPNAME_KIBANA" --protocol tcp --port 5601
   cf add-network-policy "$CGAPPNAME_FRONTEND" "$CGAPPNAME_KIBANA" --protocol tcp --port 5601
@@ -105,12 +105,16 @@ update_backend()
 {
     cd tdrs-backend || exit
     cf unset-env "$CGAPPNAME_BACKEND" "AV_SCAN_URL"
-    
+
     if [ "$CF_SPACE" = "tanf-prod" ]; then
       cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tanf-prod-clamav-rest.apps.internal:9000/scan"
     else
       # Add environment varilables for clamav
       cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tdp-clamav-nginx-$env.apps.internal:9000/scan"
+
+      # Add variable for dev/staging apps to know their DB name. Prod uses default AWS name.
+      cf unset-env "$CGAPPNAME_BACKEND" "APP_DB_NAME"
+      cf set-env "$CGAPPNAME_BACKEND" "APP_DB_NAME" "tdp_db_$backend_app_name"
     fi
 
     if [ "$1" = "rolling" ] ; then
@@ -129,12 +133,12 @@ update_backend()
     fi
 
     set_cf_envs
-    
+
     cf map-route "$CGAPPNAME_BACKEND" apps.internal --hostname "$CGAPPNAME_BACKEND"
 
     # Add network policy to allow frontend to access backend
     cf add-network-policy "$CGAPPNAME_FRONTEND" "$CGAPPNAME_BACKEND" --protocol tcp --port 8080
-    
+
     if [ "$CF_SPACE" = "tanf-prod" ]; then
       # Add network policy to allow backend to access tanf-prod services
       cf add-network-policy "$CGAPPNAME_BACKEND" clamav-rest --protocol tcp --port 9000
@@ -149,7 +153,7 @@ bind_backend_to_services() {
     echo "Binding services to app: $CGAPPNAME_BACKEND"
 
     if [ "$CGAPPNAME_BACKEND" = "tdp-backend-develop" ]; then
-      # TODO: this is technical debt, we should either make staging mimic tanf-dev 
+      # TODO: this is technical debt, we should either make staging mimic tanf-dev
       #       or make unique services for all apps but we have a services limit
       #       Introducing technical debt for release 3.0.0 specifically.
       env="develop"
@@ -158,10 +162,10 @@ bind_backend_to_services() {
     cf bind-service "$CGAPPNAME_BACKEND" "tdp-staticfiles-${env}"
     cf bind-service "$CGAPPNAME_BACKEND" "tdp-datafiles-${env}"
     cf bind-service "$CGAPPNAME_BACKEND" "tdp-db-${env}"
-    
+
     # Setting up the ElasticSearch service
     cf bind-service "$CGAPPNAME_BACKEND" "es-${env}"
-    
+
     set_cf_envs
 
     echo "Restarting app: $CGAPPNAME_BACKEND"

tdrs-backend/Dockerfile

@@ -15,7 +15,7 @@ RUN apt-get -y update
 # Upgrade already installed packages:
 RUN apt-get -y upgrade
 # Postgres client setup
-RUN apt install -y postgresql-common curl ca-certificates && install -d /usr/share/postgresql-common/pgdg && \
+RUN apt --purge remove postgresql postgresql-* && apt install -y postgresql-common curl ca-certificates && install -d /usr/share/postgresql-common/pgdg && \
 curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc && \
 sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' && \
 apt -y update && apt install postgresql-client-15 -y

tdrs-backend/tdpservice/email/email.py

@@ -6,10 +6,9 @@
 from django.conf import settings
 from django.template.loader import get_template
 from tdpservice.core.utils import log
-
 import logging
 
-logger = logging.getLogger()
+logger = logging.getLogger(__name__)
 
 
 def prepare_recipients(recipient_email):
@@ -78,6 +77,8 @@ def filter_valid_emails(emails, logger_context=None):
                 logger_context=logger_context
             )
     if len(valid_emails) == 0:
-        raise ValidationError("No valid emails provided.")
+        log("No valid emails provided.",
+            logger_context,
+            "warn")
 
     return valid_emails

tdrs-backend/tdpservice/email/helpers/data_file.py

@@ -43,7 +43,7 @@ def send_data_submitted_email(
         "url": settings.FRONTEND_BASE_URL
     }
 
-    log(f'Data file submitted; emailing Data Analysts {recipients}', logger_context=logger_context)
+    log(f'Data file submitted; emailing Data Analysts {list(recipients)}', logger_context=logger_context)
 
     match datafile_summary.status:
         case DataFileSummary.Status.PENDING:

tdrs-backend/tdpservice/email/test/test_email.py

@@ -2,7 +2,6 @@
 
 from django.core import mail
 from django.test import TestCase
-from django.core.exceptions import ValidationError
 
 from tdpservice.email.email import (
     automated_email,
@@ -50,8 +49,7 @@ def test_automated_email_fails_with_invalid_email(self):
 
         mail.outbox.clear()
 
-        with self.assertRaises(ValidationError):
-            automated_email(email_path, recipient_email, subject, email_context, text_message)
+        automated_email(email_path, recipient_email, subject, email_context, text_message)
         self.assertEqual(len(mail.outbox), 0)
 
     def test_filter_valid_emails(self):
@@ -64,5 +62,4 @@ def test_filter_valid_emails_fails(self):
         """Test validate emails raised ValidationError ."""
         emails = ["foo", "bar"]
 
-        with self.assertRaises(ValidationError):
-            filter_valid_emails(emails)
+        assert len(filter_valid_emails(emails)) == 0

tdrs-backend/tdpservice/parsers/case_consistency_validator.py

@@ -168,7 +168,7 @@ def __validate_section1(self, num_errors):
     def __validate_section2(self, num_errors):
         """Perform TANF Section 2 category four validation on all cached records."""
         num_errors += self.__validate_s2_records_are_related()
-        num_errors += self.__validate_t5_aabd_and_ssi()
+        num_errors += self.__validate_t5_atd_and_ssi()
         return num_errors
 
     def __validate_family_affiliation(self, num_errors, t1s, t2s, t3s, error_msg):
@@ -390,7 +390,7 @@ def __validate_s2_records_are_related(self):
                 num_errors += 1
         return num_errors
 
-    def __validate_t5_aabd_and_ssi(self):
+    def __validate_t5_atd_and_ssi(self):
         num_errors = 0
         is_ssp = self.program_type == 'SSP'
 
@@ -403,7 +403,7 @@ def __validate_t5_aabd_and_ssi(self):
         t5s = self.sorted_cases.get(t5_model, [])
 
         for record, schema in t5s:
-            rec_aabd = getattr(record, 'REC_AID_TOTALLY_DISABLED')
+            rec_atd = getattr(record, 'REC_AID_TOTALLY_DISABLED')
             rec_ssi = getattr(record, 'REC_SSI')
             family_affiliation = getattr(record, 'FAMILY_AFFILIATION')
             dob = getattr(record, 'DATE_OF_BIRTH')
@@ -413,7 +413,7 @@ def __validate_t5_aabd_and_ssi(self):
             dob_date = datetime.strptime(dob, '%Y%m%d')
             is_adult = get_years_apart(rpt_date, dob_date) >= 19
 
-            if is_territory and is_adult and (rec_aabd != 1 and rec_aabd != 2):
+            if is_territory and is_adult and rec_atd not in {1, 2}:
                 self.__generate_and_add_error(
                     schema,
                     record,
@@ -424,7 +424,7 @@ def __validate_t5_aabd_and_ssi(self):
                     )
                 )
                 num_errors += 1
-            elif is_state and rec_aabd != 2:
+            elif is_state and rec_atd == 1:
                 self.__generate_and_add_error(
                     schema,
                     record,
@@ -446,7 +446,7 @@ def __validate_t5_aabd_and_ssi(self):
                     )
                 )
                 num_errors += 1
-            elif is_state and family_affiliation == 1:
+            elif is_state and family_affiliation == 1 and rec_ssi not in {1, 2}:
                 self.__generate_and_add_error(
                     schema,
                     record,

tdrs-backend/tdpservice/parsers/fields.py

@@ -18,6 +18,7 @@ def __init__(
         endIndex,
         required=True,
         validators=[],
+        ignore_errors=False,
     ):
         self.item = item
         self.name = name
@@ -27,6 +28,7 @@ def __init__(
         self.endIndex = endIndex
         self.required = required
         self.validators = validators
+        self.ignore_errors = ignore_errors
 
     def create(self, item, name, length, start, end, type):
         """Create a new field."""
@@ -64,7 +66,7 @@ class TransformField(Field):
     """Represents a field that requires some transformation before serializing."""
 
     def __init__(self, transform_func, item, name, friendly_name, type, startIndex, endIndex, required=True,
-                 validators=[], **kwargs):
+                 validators=[], ignore_errors=False, **kwargs):
         super().__init__(
             item=item,
             name=name,
@@ -73,7 +75,8 @@ def __init__(self, transform_func, item, name, friendly_name, type, startIndex,
             startIndex=startIndex,
             endIndex=endIndex,
             required=required,
-            validators=validators)
+            validators=validators,
+            ignore_errors=ignore_errors)
         self.transform_func = transform_func
         self.kwargs = kwargs
 

tdrs-backend/tdpservice/parsers/models.py

@@ -6,6 +6,9 @@
 from django.contrib.contenttypes.fields import GenericForeignKey
 from django.contrib.contenttypes.models import ContentType
 from tdpservice.data_files.models import DataFile
+import logging
+
+logger = logging.getLogger(__name__)
 
 class ParserErrorCategoryChoices(models.TextChoices):
     """Enum of ParserError error_type."""
@@ -93,8 +96,29 @@ class Status(models.TextChoices):
     total_number_of_records_in_file = models.IntegerField(null=True, blank=False, default=0)
     total_number_of_records_created = models.IntegerField(null=True, blank=False, default=0)
 
+    def set_status(self, status):
+        """Set the status on the summary object."""
+        match status:
+            case DataFileSummary.Status.PENDING:
+                self.status = DataFileSummary.Status.PENDING
+            case DataFileSummary.Status.ACCEPTED:
+                self.status = DataFileSummary.Status.ACCEPTED
+            case DataFileSummary.Status.ACCEPTED_WITH_ERRORS:
+                self.status = DataFileSummary.Status.ACCEPTED_WITH_ERRORS
+            case DataFileSummary.Status.PARTIALLY_ACCEPTED:
+                self.status = DataFileSummary.Status.PARTIALLY_ACCEPTED
+            case DataFileSummary.Status.REJECTED:
+                self.status = DataFileSummary.Status.REJECTED
+            case _:
+                logger.warn(f"Unknown status: {status} passed into set_status.")
+
     def get_status(self):
         """Set and return the status field based on errors and models associated with datafile."""
+        # Because we introduced a setter for the status for exception handling, we need to
+        # check if it has been set before determining a status based on the queries below.
+        if self.status != DataFileSummary.Status.PENDING:
+            return self.status
+
         errors = ParserError.objects.filter(file=self.datafile)
 
         # excluding row-level pre-checks and trailer pre-checks.