Merge pull request #2426 from HHS/main

[Prod] Updates to dev environment, add standard goals checkbox to ARs, correct national center name in historical data, fix bug with objective duplication

.circleci/config.yml

@@ -563,7 +563,7 @@ parameters:
     default: "al-ttahub-3329-fix-event-view"
     type: string
   sandbox_git_branch: # change to feature branch to test deployment
-    default: "kw-fix-duplicate-programs"
+    default: "mb/TTAHUB-3483/checkbox-to-activity-reports"
     type: string
   prod_new_relic_app_id:
     default: "877570491"

Dockerfile

@@ -1,3 +1,10 @@
 FROM node:18.20.3
 WORKDIR /app
 RUN apt-get update && apt-get install lcov -y
+RUN \
+    mkdir -p /home/node/.cache/yarn \
+    && chown -R node:node /home/node/.cache/yarn \
+    && mkdir -p /app/node_modules \
+    && chown -R node:node /app/node_modules
+
+

bin/prod-style-server

@@ -25,20 +25,20 @@ fi
 
 # Run a production style build (single BE server with static FE)
 echo "Running yarn install on server..."
-docker-compose -f docker-compose.dss.yml run --rm server yarn install --production=false
+docker compose -f docker-compose.dss.yml run --rm server yarn install --production=false
 echo "Running yarn install on frontend..."
-docker-compose -f docker-compose.dss.yml run --rm server yarn --cwd frontend install --production=false
+docker compose -f docker-compose.dss.yml run --rm server yarn --cwd frontend install --production=false
 echo "Building backend..."
-docker-compose -f docker-compose.dss.yml run --rm server yarn build
+docker compose -f docker-compose.dss.yml run --rm server yarn build
 echo "Building frontend..."
-docker-compose -f docker-compose.dss.yml run --rm server yarn --cwd frontend run build
+docker compose -f docker-compose.dss.yml run --rm server yarn --cwd frontend run build
 
 # Starting services
 echo "Starting services..."
-docker-compose -f docker-compose.dss.yml up -d
+docker compose -f docker-compose.dss.yml up -d
 
 # Running database migrations
 echo "Running database migrations..."
-docker-compose -f docker-compose.dss.yml run --rm server yarn db:migrate:ci
+docker compose -f docker-compose.dss.yml run --rm server yarn db:migrate:ci
 
 echo "Server setup completed successfully."

bin/run-tests

@@ -9,7 +9,7 @@ log() {
 
 check_exit() {
     if [[ "$1" -ne 0 ]]; then
-        echo "$lr last docker-compose command failed"
+        echo "$lr last docker compose command failed"
         ((exit_code++))
     fi
 }
@@ -42,19 +42,19 @@ main() {
       esac
     done
 
-    docker-compose \
+    docker compose \
         -f 'docker-compose.test.yml' \
         down --volumes --remove-orphans
 
     log "Running tests in using test config 'docker-compose.test.yml'"
     # Start containers
     if [[ $docker == "true" ]]; then
-      docker-compose -f 'docker-compose.test.yml' up -d
+      docker compose -f 'docker-compose.test.yml' up -d
       check_exit "$?"
     else
-      docker-compose -f 'docker-compose.test.yml' up -d test-db
+      docker compose -f 'docker-compose.test.yml' up -d test-db
       check_exit "$?"
-      IFS=: read host port <<< "$(docker-compose -f 'docker-compose.test.yml' port test-db 5432)"
+      IFS=: read host port <<< "$(docker compose -f 'docker-compose.test.yml' port test-db 5432)"
       echo "Postgres is running on port $port"
       export POSTGRES_PORT=$port
     fi
@@ -177,7 +177,7 @@ main() {
     # Cleanup
     echo
     log "Cleaning up test containers"
-    docker-compose \
+    docker compose \
         -f 'docker-compose.test.yml' \
         down --volumes --remove-orphans
     check_exit "$?"

bin/test-backend-ci

@@ -4,7 +4,7 @@ declare -i exit_code=0
 
 check_exit() {
     if [[ "$1" -ne 0 ]]; then
-        echo "$lr last docker-compose command failed"
+        echo "$lr last docker compose command failed"
         ((exit_code++))
     fi
 }

docker-compose.debug.yml

@@ -1,4 +1,3 @@
-version: "3.7"
 services:
   backend:
     build:
@@ -28,6 +27,8 @@ services:
       - PUPPETEER_SKIP_CHROMIUM_DOWNLOAD
     volumes:
       - ".:/app:rw"
+      - "node_modules-backend:/app/node_modules:rw"
+      - "yarn-cache:/home/node/.cache/yarn:rw"
     deploy:
       resources:
         limits:
@@ -43,6 +44,8 @@ services:
     volumes:
       - "./frontend:/app:rw"
       - "./scripts:/app/scripts"
+      - "node_modules-frontend:/app/node_modules:rw"
+      - "yarn-cache:/home/node/.cache/yarn:rw"
     environment:
       - BACKEND_PROXY=http://backend:8080
   worker:
@@ -59,3 +62,5 @@ services:
       - SMTP_HOST=mailcatcher
     volumes:
       - ".:/app:rw"
+      - "node_modules-backend:/app/node_modules:rw"
+      - "yarn-cache:/home/node/.cache/yarn:rw"

docker-compose.dss.yml

@@ -1,4 +1,3 @@
-version: "3.2"
 services:
   server:
     build:

docker-compose.override.yml

@@ -1,8 +1,9 @@
-version: "3.7"
 services:
   backend:
     build:
       context: .
+    profiles:
+      - minimal_required
     command: yarn server
     user: ${CURRENT_USER:-root}
     ports:
@@ -28,9 +29,13 @@ services:
       - PUPPETEER_SKIP_CHROMIUM_DOWNLOAD
     volumes:
       - ".:/app:rw"
+      - "node_modules-backend:/app/node_modules:rw"
+      - "yarn-cache:/home/node/.cache/yarn:rw"
   frontend:
     build:
       context: .
+    profiles:
+      - minimal_required
     command: yarn start
     user: ${CURRENT_USER:-root}
     stdin_open: true
@@ -40,12 +45,16 @@ services:
       - "./frontend:/app:rw"
       - "./scripts:/app/scripts"
       - "./packages:/packages:ro"
+      - "node_modules-frontend:/app/node_modules:rw"
+      - "yarn-cache:/home/node/.cache/yarn:rw"
     environment:
       - BACKEND_PROXY=http://backend:8080
       - REACT_APP_WEBSOCKET_URL
   worker:
     build:
       context: .
+    profiles:
+      - minimal_required
     command: yarn worker
     env_file: .env
     depends_on:
@@ -57,8 +66,12 @@ services:
       - SMTP_HOST=mailcatcher
     volumes:
       - ".:/app:rw"
+      - "node_modules-backend:/app/node_modules:rw"
+      - "yarn-cache:/home/node/.cache/yarn:rw"
   owasp_zap_backend:
     image: softwaresecurityproject/zap-stable:latest
+    profiles:
+      - full_stack
     platform: linux/arm64
     user: zap
     command: zap-full-scan.py -t http://backend:8080 -c zap.conf  -i -r owasp_report_.html
@@ -69,6 +82,8 @@ services:
       - backend
   owasp_zap_similarity:
     image: softwaresecurityproject/zap-stable:latest
+    profiles:
+      - full_stack
     platform: linux/arm64
     user: zap
     command: zap-api-scan.py -t http://similarity:8080/openapi.json -f openapi -I -i -r owasp_api_report.html

docker-compose.test.yml

@@ -1,4 +1,3 @@
-version: "3.5"
 services:
   test-backend:
     build:
@@ -15,6 +14,8 @@ services:
       - "${ITAMS_MD_HOST:-sftp.itams.ohs.acf.hhs.gov}:0.0.0.0"
     volumes:
       - ".:/app:rw"
+      - "node_modules-backend:/app/node_modules:rw"
+      - "yarn-cache:/home/node/.cache/yarn:rw"
     networks:
       - ttadp-test
   test-frontend:
@@ -27,6 +28,8 @@ services:
     volumes:
       - "./frontend:/app:rw"
       - "./scripts:/app/scripts"
+      - "node_modules-frontend:/app/node_modules:rw"
+      - "yarn-cache:/home/node/.cache/yarn:rw"
     environment:
       - BACKEND_PROXY=http://test-backend:8080
       - REACT_APP_WEBSOCKET_URL

docker-compose.yml

@@ -1,7 +1,8 @@
-version: "3.7"
 services:
   api-docs:
     image: redocly/redoc
+    profiles:
+      - full_stack
     ports:
       - "5003:80"
     volumes:
@@ -12,13 +13,17 @@ services:
     image: postgres:15.6
     container_name: postgres_docker
     env_file: .env
+    profiles:
+      - minimal_required
     ports:
       - "5432:5432"
     volumes:
       - dbdata:/var/lib/postgresql/data
     shm_size: 1g
   minio:
     image: minio/minio:RELEASE.2024-01-01T16-36-33Z
+    profiles:
+      - full_stack
     env_file: .env
     ports:
       - "9000:9000"
@@ -28,19 +33,25 @@ services:
     command: server /data --console-address ":9001"
   aws-cli:
     image: amazon/aws-cli
+    profiles:
+      - full_stack
     env_file: .env
     command: ["--endpoint-url", "http://minio:9000", "s3api", "create-bucket", "--bucket", "$S3_BUCKET"]
     depends_on:
       - minio
   clamav-rest:
     image: ajilaag/clamav-rest
+    profiles:
+      - full_stack
     ports:
       - "9443:9443"
     environment:
       - MAX_FILE_SIZE=30M
   similarity_api:
     build:
       context: ./similarity_api
+    profiles:
+      - minimal_required
     ports:
       - "9100:8080"
     env_file: .env
@@ -50,18 +61,24 @@ services:
       - "./similarity_api/src:/app:rw"
   redis:
     image: redis:5.0.6-alpine
+    profiles:
+      - minimal_required
     command: ['redis-server', '--requirepass', '$REDIS_PASS']
     env_file: .env
     ports:
       - "6379:6379"
   mailcatcher:
     image: schickling/mailcatcher
+    profiles:
+      - full_stack
     ports:
       - "1025:1025"
       - "1080:1080"
   testingonly:
     build:
       context: .
+    profiles:
+      - full_stack
     ports:
       - "9999:9999"
     depends_on:
@@ -74,6 +91,8 @@ services:
       - NODE_ENV=development
   sftp:
     image: jmcombs/sftp:alpine
+    profiles:
+      - full_stack
     volumes:
         - ./test-sftp:/home/tta_ro/ProdTTAHome
         - ./test-sftp/sshd_config:/etc/ssh/sshd_config
@@ -83,3 +102,6 @@ services:
 volumes:
   dbdata: {}
   minio-data: {}
+  yarn-cache: {}
+  node_modules-backend: {}
+  node_modules-frontend: {}

docs/testing.md

@@ -38,11 +38,11 @@ By default, `./bin/run-tests` will run both backend and frontend tests. If you w
 
 When running tests in Docker, be aware that there are tests that will modify/delete database records. For tests to run, the 'db' service needs to exist and `db:migrate` and `db:seed` need to have been run (to create the tables and populate certain records).
 
-In the `docker-compose.yml` configuration, the database is set up to persist to a volume, "dbdata", so database records will persist between runs of the 'db' service, unless you remove that volume explicitly (e.g. `docker volume rm` or `docker-compose down --volumes`).
+In the `docker-compose.yml` configuration, the database is set up to persist to a volume, "dbdata", so database records will persist between runs of the 'db' service, unless you remove that volume explicitly (e.g. `docker volume rm` or `docker compose down --volumes`).
 
 
-### Notes on docker-compose and multiple configurations
+### Notes on docker compose and multiple configurations
 
-`docker-compose` has a feature for providing multiple `docker-compose.*.yml` files where subsequent files can override settings in previous files, which sounds like it would suit the use case of running docker for local development and for testing. However, the ability to [override configurations](https://docs.docker.com/compose/extends/#adding-and-overriding-configuration) is limited. While experimenting with overrides, it became clear that doing so would require a minimum of three docker-compose.yml files: one "base", one for local development, one for running tests. Trying to compose docker-compose.yml files would be complicated.
+`docker compose` has a feature for providing multiple `docker-compose.*.yml` files where subsequent files can override settings in previous files, which sounds like it would suit the use case of running docker for local development and for testing. However, the ability to [override configurations](https://docs.docker.com/compose/extends/#adding-and-overriding-configuration) is limited. While experimenting with overrides, it became clear that doing so would require a minimum of three docker-compose.yml files: one "base", one for local development, one for running tests. Trying to compose docker-compose.yml files would be complicated.
 
 In addition, while experimenting with multiple configuration files, it became clear that docker was unable to differentiate between different versions of the same service. Trying to override the 'db' service for testing would not work as expected: if the local/dev 'db' service had already been created, that one would be used when tests were run.

frontend/package.json

@@ -25,7 +25,7 @@
     "follow-redirects": "^1.15.6",
     "html-to-draftjs": "^1.5.0",
     "html2canvas": "^1.4.1",
-    "http-proxy-middleware": "^2.0.1",
+    "http-proxy-middleware": "^2.0.7",
     "lodash": "^4.17.20",
     "minimist": "^1.2.6",
     "moment": "^2.29.4",