Release/v3.4.0 sprint 98

.circleci/build-and-test/jobs.yml

@@ -1,10 +1,9 @@
-# jobs:
   test-backend:
     executor: machine-executor
     steps:
       - checkout
       - docker-compose-check
-      - docker-compose-up-with-elastic-backend
+      - docker-compose-up-backend
       - run:
           name: Run Unit Tests And Create Code Coverage Report
           command: |
@@ -47,7 +46,7 @@
     steps:
       - checkout
       - docker-compose-check
-      - docker-compose-up-with-elastic-backend
+      - docker-compose-up-backend
       - docker-compose-up-frontend
       - install-nodejs-machine
       - disable-npm-audit

.circleci/build-and-test/workflows.yml

@@ -5,7 +5,7 @@
       - secrets-check
       - test-backend:
           requires:
-            - secrets-check      
+            - secrets-check
       - test-frontend:
           requires:
             - secrets-check
@@ -30,7 +30,7 @@
                 - master
                 - /^release.*/
           requires:
-            - secrets-check      
+            - secrets-check
       - test-frontend:
           filters:
             branches:

.circleci/deployment/commands.yml

@@ -95,8 +95,8 @@
           name: Install dependencies
           command: |
             sudo apt update
-            sudo add-apt-repository ppa:rmescandon/yq
-            sudo apt-get install yq
+            sudo apt-get install -y wget
+            sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/v4.42.1/yq_linux_amd64 && chmod +x /usr/local/bin/yq
       - run:
           name: Apply database migrations
           command: |
@@ -275,7 +275,7 @@
           cf-space: <<parameters.cf-space>>
       - run:
           name: Ensure s3 versioning key exists
-          # use cf to check if a service key exists for tdp-datafiles-dev(or prod) 
+          # use cf to check if a service key exists for tdp-datafiles-dev(or prod)
           # exit 0 if it does, create it if it doesn't
           command: |
             chmod +x ./scripts/create_s3_versioning_key.sh

.circleci/deployment/jobs.yml

@@ -1,4 +1,3 @@
-# jobs:
   deploy-dev:
     parameters:
       target_env:

.circleci/owasp/jobs.yml

@@ -1,4 +1,3 @@
-# jobs:
   backend-owasp-scan:
     executor: large-machine-executor
     working_directory: ~/tdp-apps
@@ -85,23 +84,23 @@
           command: |
             # Construct the project slug from the current branch name and user
             PROJECT_SLUG=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
-
             # These environment variables are exported to Circle CI's BASH_ENV
             # by the zap-scanner.sh script for each respective app target.
             CMD_ARGS=(
               "$CIRCLE_BUILD_NUM"
-              --backend-pass-count ${ZAP_BACKEND_PASS_COUNT:-0}
-              --backend-warn-count ${ZAP_BACKEND_WARN_COUNT:-0}
-              --backend-fail-count ${ZAP_BACKEND_FAIL_COUNT:-0}
-              --frontend-pass-count ${ZAP_FRONTEND_PASS_COUNT:-0}
-              --frontend-warn-count ${ZAP_FRONTEND_WARN_COUNT:-0}
-              --frontend-fail-count ${ZAP_FRONTEND_FAIL_COUNT:-0}
-              --project-slug $PROJECT_SLUG
+              --backend-pass-count "${ZAP_BACKEND_PASS_COUNT:-0}"
+              --backend-warn-count "${ZAP_BACKEND_WARN_COUNT:-0}"
+              --backend-fail-count "${ZAP_BACKEND_FAIL_COUNT:-0}"
+              --frontend-pass-count "${ZAP_FRONTEND_PASS_COUNT:-0}"
+              --frontend-warn-count "${ZAP_FRONTEND_WARN_COUNT:-0}"
+              --frontend-fail-count "${ZAP_FRONTEND_FAIL_COUNT:-0}"
+              --project-slug "$PROJECT_SLUG"
             )
             # Evaluate the full command before passing it in so it doesn't
             # get improperly interpolated by Cloud.gov.
-            CMD="python manage.py process_owasp_scan ${CMD_ARGS[@]}"
-            # Submit a CF Task for execution that will run the necessary command
+            CMD="python manage.py process_owasp_scan ${CMD_ARGS[*]}"
+            # Submit a CF Task for execution after a 4 minute sleep to ensure all of the scan's previous state has been closed.
+            sleep 240
             cf run-task tdp-backend-<< parameters.target_env >> \
               --command "$CMD" \
               --name nightly-owasp-scan

.circleci/util/commands.yml

@@ -11,12 +11,6 @@
           name: Build and spin-up Django API service
           command: cd tdrs-backend; docker network create external-net; docker-compose up -d --build
 
-  docker-compose-up-with-elastic-backend:
-    steps:
-      - run:
-          name: Build and spin-up Django API service
-          command: cd tdrs-backend; docker network create external-net; docker-compose --profile elastic_setup up -d --build
-
   cf-check:
     steps:
       - run:

.circleci/util/jobs.yml

@@ -1,4 +1,3 @@
-# jobs:
   make_erd:
     executor: machine-executor
     working_directory: ~/tdp_apps
@@ -9,7 +8,7 @@
           name: Run graph_models
           command: |
             cd tdrs-backend
-            if [ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]; then 
+            if [ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]; then
             docker network create external-net
             fi
             docker-compose run --rm web bash -c \

CODE_OF_CONDUCT.md

@@ -0,0 +1,45 @@
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, caste, color, religion, or sexual identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our community include:
+
+- Demonstrating empathy and kindness toward other people
+- Being respectful of differing opinions, viewpoints, and experiences
+- Giving and gracefully accepting constructive feedback
+- Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience
+- Focusing on what is best not just for us as individuals, but for the overall community
+
+Examples of unacceptable behavior include:
+
+- The use of sexualized language or imagery, and sexual attention or advances of any kind
+- Trolling, insulting or derogatory comments, and personal or political attacks
+- Public or private harassment
+- Publishing others’ private information, such as a physical or email address, without their explicit permission
+- Other conduct which could reasonably be considered inappropriate in a professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official email address, posting via an official social media account, or acting as an appointed representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to project points of contact listed in the [project readme](README.md). All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the reporter of any incident.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.1, available at
+[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].

CONTRIBUTING.md

@@ -0,0 +1,59 @@
+# How to Contribute
+
+Thank you for taking the time to contribute! 
+
+We're so thankful you're considering contributing to an [open source project of the U.S. government](https://code.gov/)! If you're unsure about anything, please feel free to reach out to the points of contact documented in the project readme. The worst that can happen is you'll be politely asked to change something. We appreciate all friendly contributions.
+
+We encourage you to read this project's [LICENSE](LICENSE.md), [CODE_OF_CONDUCT](CODE_OF_CONDUCT.md), and its [README](README.md).
+
+## How Can I Contribute?
+
+There are a number of ways to contribute to this project.
+
+### Report a Bug
+
+If you do not find your bug in our issues list, file a bug report. When reporting the bug, please follow these guidelines:
+
+- **Please use the [Bug Report](https://github.com/HHS/TANF-app/issues/new?assignees=&labels=dev&projects=&template=bug-template.md&title=) issue template** This is populated with information and questions that will help TANF Data Portal (TDP) developers resolve the issue
+- **Use a clear and descriptive issue title** for the issue to identify the problem.
+- **Describe the exact steps to reproduce the problem** in as much detail as possible. For example, start by explaining how you got to the page where you encountered the bug and what you were attempting to do when the bug occurred.
+- **Describe the behavior you observed after following the steps** and point out what exactly is the problem with that behavior.
+- **Explain which behavior you expected to see instead and why.**
+- **Include screenshots, animated GIFs, or videos** if possible, which show you following the described steps and clearly demonstrate the problem.
+- **If the problem wasn't triggered by a specific action**, describe what you were doing before the problem happened.
+
+### Suggest an Enhancement
+
+If you don't have specific language or code to submit but would like to suggest a change, request a feature, or have something addressed, you can open an issue in this repository.
+
+In this issue, please describe the use case for the feature you would like to see -, what you need, why you need it, and how it should work. Team members will respond to the Feature request as soon as possible. Often, Feature request suggestions undergo a collaborative discussion with the community to help refine the need for the feature and how it can be implemented.
+
+## Code Contributions
+
+### Getting Started
+
+Please refer to readmes in the [project repo's root](https://github.com/HHS/TANF-app/blob/main/README.md), [how we work](https://github.com/HHS/TANF-app/blob/main/docs/How-We-Work/README.md), and [technical documentation](https://github.com/HHS/TANF-app/blob/main/docs/Technical-Documentation/README.md) folders for detailed information on contributing to the project. Please see [our-workflow.md](https://github.com/HHS/TANF-app/blob/main/docs/How-We-Work/our-workflow.md) for details on current PR processes. 
+
+## Policies
+
+### Security and Responsible Disclosure Policy
+
+The Department of Health and Human Services is committed to ensuring the security of the American public by protecting their information from
+unwarranted disclosure. We want security researchers to feel comfortable reporting vulnerabilities they have discovered so we can fix them and keep our users safe. We developed our disclosure policy to reflect our values and uphold our sense of responsibility to security researchers who share their expertise with us in good faith.
+
+_Submit a vulnerability:_ Unfortunately, we cannot accept secure submissions via email or via GitHub Issues. Please use our website to submit vulnerabilities at [https://hhs.responsibledisclosure.com](https://hhs.responsibledisclosure.com). HHS maintains an acknowledgements page to recognize your efforts on behalf of the American public, but you are also welcome to submit anonymously.
+
+Review the HHS Disclosure Policy and websites in scope:
+[https://www.hhs.gov/vulnerability-disclosure-policy/index.html](https://www.hhs.gov/vulnerability-disclosure-policy/index.html).
+
+This policy describes _what systems and types of research_ are covered under this policy, _how to send_ us vulnerability reports, and _how long_ we ask security researchers to wait before publicly disclosing vulnerabilities.
+
+### Public domain
+
+This project is in the public domain within the United States, and copyright and related rights in the work worldwide are waived through the [CC0 1.0 Universal public domain dedication](https://creativecommons.org/publicdomain/zero/1.0/).
+
+All contributions to this project will be released under the CC0 dedication. By submitting a pull request or issue, you are agreeing to comply with this waiver of copyright interest.
+
+### ACF Privacy Policy
+
+TDP operates under [ACF's Privacy Policy](https://www.acf.hhs.gov/privacy-policy). 

LICENSE → LICENSE.md

@@ -28,4 +28,4 @@ Unless expressly stated otherwise, the person who associated a work with
 this deed makes no warranties about the work, and disclaims liability for
 all uses of the work, to the fullest extent permitted by applicable law.
 When using or citing the work, you should not imply endorsement by the
-author or the affirmer.
+author or the affirmer.

README.md

@@ -79,4 +79,4 @@ TDP is developed to be (at minimum) compliant with Section 508 of the Rehabilita
 |--|--|--|--|
 | Product Owner |Lauren Frohlich |@lfrohlich |[email protected]|
 | Government Technical Monitor |Alex Pennington |@adpennington |[email protected]|
-| Vendor Product Manager |Connor Smith |@Smithh-Co |csmith@teamraft.com |
+| Vendor Product Manager |Rob Gendron |@robgendron |rgendron@teamraft.com |

docs/Sprint-Review/sprint-95-summary.md

@@ -0,0 +1,91 @@
+# Sprint 95 Summary
+
+03/13/2024 - 03/26/2024
+
+Velocity (Dev): 19
+
+## Sprint Goal
+* Dev:
+    * Category 4 Validators #2842
+    * Further validation enhancements #2757, #2807, #2818, ... 
+
+* DevOps:
+    * ES re-indexing #2870
+    * ES aggregation #2814
+
+* Design: 
+    * Error Categories GitHub Documentation
+    * Friendly name fixes (#2801)
+    * Knowledge Center Content (#2847, #2846)
+        * Submission History
+        * Error reports / data file structure
+
+
+
+## Tickets
+### Completed/Merged
+* [#2746 As an STT, I need to know if there are issues with the DOBs reported in my data files](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2746)
+* [#2729 As a developer, I want to move migration commands in the pipeline to CircleCI](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2729)
+* [#2820 [bug] Uncaught exception re: parsing error preventing feedback report generation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2820)
+* [#2799 Generate error mismatching field rpt_month_year w/ header](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2799)
+* [#2793 update dateYearIsLargerThan() validator logic for rpt_month_year](https://github.com/raft-tech/TANF-app/issues/2793)
+* [#2754 As tech lead I need sections 3,4 rejected if T6, T7, M7 records can't be parsed](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2754)
+* [V3.3.2 yq hotfix](https://github.com/raft-tech/TANF-app/pull/2895)
+* [#2861 DIGIT Team Group + Kibana + Queries](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2861)
+
+### Ready to Merge
+* [Move ES AWS routing image](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2877)
+
+
+
+
+
+
+### Submitted (QASP Review, OCIO Review)
+* [#2536 [spike] Cat 4 validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2536)
+* [#2807 Update validation logic for CALENDAR_Quarter field](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2807)
+* [#2681 TANF Section 1 Validation cleanup](https://github.com/raft-tech/TANF-app/issues/2681)
+* [#1441 As tech lead I need new permissions group for OFA data analysts](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1441)
+* [#2871 As tech lead I need file transfer bug resolved](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2871)
+* [#2818 I need TDP to reject files that do not have an update indicator](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2818)
+* [#2886 Bug SSP feedback reports not downloadable](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2886)
+
+
+
+
+
+### Closed (not merged)
+
+
+
+---
+
+## Moved to Next Sprint (In Progress, Blocked, Raft Review)
+### In Progress
+* [#2673 Cat 1 errors audit fixes](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2673)
+* [#2509 As a data analyst I need to know when my data has been processed with or w/o errors](https://github.com/raft-tech/TANF-app/issues/2509)
+* [#2801 Friendly Name Cleanup](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2801)
+* [#2846 Submission History Knowledge Center Explainer](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2846)
+* [#2847 Error Report Knowledge Center Explainer](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2847)
+* [#2845 GitHub Error Categories guide](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2845)
+* [#2870 As tech lead I need elastic re-indexing to be automated](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2870)
+
+
+
+### Blocked
+* [#2768 Fix production OWASP scan reporting](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2768)
+* [#2592 Deploy celery as a separate cloud.gov app](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2592)
+
+
+### Raft Review
+* [#2757 Generate preparser errors when multirecord rows are the wrong length](https://github.com/raft-tech/TANF-app/issues/2757)
+* [#2842 Complete cat4 validation implementation](https://github.com/raft-tech/TANF-app/issues/2842)
+* [#2814 Aggregate cloud.gov ES instances](https://github.com/raft-tech/TANF-app/issues/2814)
+* [#2887 SSP Section 2 item # 18A (REC_OASDI_INSURANCE) schema def](https://github.com/raft-tech/TANF-app/issues/2887)
+
+
+
+
+
+
+