DBT Platform migration

.copilot/config.yml

@@ -0,0 +1,4 @@
+repository: data-hub-frontend
+builder:
+  name: paketobuildpacks/builder-jammy-full
+  version: 0.3.339

README.md

@@ -114,12 +114,17 @@ This method is recommended if you are only making frontend changes.
 
 5.  Go to the [Playbook](https://readme.trade.gov.uk/docs/playbooks/datahub.html#environments), find the environment you want to use in the Admin URLs section, and set this URL to the environment variable `API_ROOT`. You will need to remove `/admin` from the end of the API path.
 
-6.  Go to Vault, look in datahub-fe, and click on the environment you want to use. Change the following environment variables in your `.env` file to the ones specified in Vault:
+6.  Go to the parameter store in AWS (https://eu-west-2.console.aws.amazon.com/systems-manager/parameters/?region=eu-west-2&tab=Table). Change the following environment variables in your `.env` file to the ones specified in the parameter store. Remember to select the correct envriorment (e.g dev, uat, staging) you are using.
 
-    ```
-    DATA_HUB_BACKEND_ACCESS_KEY_ID=frontend-key-id
-    DATA_HUB_BACKEND_SECRET_ACCESS_KEY=frontend-key
-    ```
+> **Warning**
+> Gov PaaS will be deprecated from July 2024 and keys will be moved from Vault to Parameter Store.
+>
+> Go to Vault, look in datahub-fe, and click on the environment you want to use. Change the following environment variables in your `.env` file to the ones specified in Vault:
+
+```
+DATA_HUB_BACKEND_ACCESS_KEY_ID=frontend-key-id
+DATA_HUB_BACKEND_SECRET_ACCESS_KEY=frontend-key
+```
 
 7.  The environment variables copied from `sample.env` are set up for running both the frontend and the API using the docker. To run the frontend natively, the following variables will need to be changed to:
 
@@ -190,7 +195,7 @@ These instructions work for both the dockerised environment and the native envir
 
 List of all environment variables can be found in the source code of [envSchema.js](./src/config/envSchema.js).
 
-Check Vault for environment variables that point to other environments, such as staging and dev.
+Check AWS parameter store for environment variables that point to other environments, such as staging and dev.
 
 ## Making changes
 

docker-compose.base.yml

@@ -26,4 +26,4 @@ services:
       NODE_ENV: test
     command: bash -c 'npm run build:for-test-coverage && npm run start:coverage'
   redis:
-    image: redis:6.2.6
+    image: redis:7.2.4

docker-compose.e2e.backend.yml

@@ -5,7 +5,7 @@ services:
       - api
 
   api:
-    image: gcr.io/sre-docker-registry/github.com/uktrade/data-hub-api:main
+    image: gcr.io/sre-docker-registry/github.com/uktrade/data-hub-api:migration-deploy
     env_file: .env
     environment:
       DEBUG: 'False'
@@ -20,9 +20,9 @@ services:
       ACTIVITY_STREAM_OUTGOING_ACCESS_KEY_ID: incoming-some-id-1
       ACTIVITY_STREAM_OUTGOING_SECRET_ACCESS_KEY: incoming-some-secret-1
       ACTIVITY_STREAM_ACCESS_KEY_ID: some-id
-      ACTIVITY_STREAM_INCOMING_ACCESS_KEY_ID: some-id
       ACTIVITY_STREAM_SECRET_ACCESS_KEY: some-secret
       ACTIVITY_STREAM_INCOMING_SECRET_ACCESS_KEY: some-secret
+      DATABASE_CREDENTIALS: '{"username": "postgres", "password": "datahub", "engine": "postgres", "port": 5432, "dbname": "datahub", "host": "postgres", "dbInstanceIdentifier": "db-instance"}'
     ports:
       - 8000:8000
     depends_on:
@@ -35,30 +35,29 @@ services:
     command: /app/setup-uat.sh || echo "all good"
 
   rq:
-    image: gcr.io/sre-docker-registry/github.com/uktrade/data-hub-api:main
+    image: gcr.io/sre-docker-registry/github.com/uktrade/data-hub-api:migration-deploy
     env_file: .env
     depends_on:
       - postgres
       - opensearch
       - redis
     entrypoint: dockerize -wait tcp://postgres:5432 -wait tcp://opensearch:9200 -wait tcp://redis:6379 -timeout 5m
     command: python short-running-worker.py long-running-worker.py
-    logging:
-      driver: none
+    environment:
+      DATABASE_CREDENTIALS: '{"username": "postgres", "password": "datahub", "engine": "postgres", "port": 5432, "dbname": "datahub", "host": "postgres", "dbInstanceIdentifier": "db-instance"}'
 
   postgres:
-    image: postgres:12
+    image: postgres:16
     ports:
-      - "5432:5432"
+      - '5432:5432'
     environment:
       POSTGRES_DB: datahub
       POSTGRES_USER: user
       POSTGRES_PASSWORD: password
-    logging:
-      driver: none
+      DATABASE_CREDENTIALS: '{"username": "postgres", "password": "datahub", "engine": "postgres", "port": 5432, "dbname": "datahub", "host": "postgres", "dbInstanceIdentifier": "db-instance"}'
 
   opensearch:
-    image: opensearchproject/opensearch:1.2.4
+    image: opensearchproject/opensearch:2.11.0
     ports:
       - '9200:9200'
       - '9300:9300'
@@ -75,7 +74,7 @@ services:
   activity-feed-reverseproxy:
     build: ./test/end-to-end/proxy
     ports:
-        - 8081:8081
+      - 8081:8081
     depends_on:
       - activity-feed
     logging:
@@ -92,7 +91,8 @@ services:
       - 8082:8082
     environment:
       - PORT=8082
-      - 'VCAP_SERVICES={"redis":[{"credentials":{"uri":"redis://redis:6379"}}],"opensearch":[{"credentials":{"uri":"http://some-id:some-secret@opensearch:9200"}, "plan": "123-123-123" } ] }'
+      - REDIS_URL=redis://redis:6379
+      - OPENSEARCH=http://some-id:some-secret@opensearch:9200
       - SENTRY_ENVIRONMENT=test
       - SENTRY_DSN=http://abc:cvb@localhost:9872/123
       - FEEDS__1__UNIQUE_ID=verification_feed_app

docker-compose.services.yml

@@ -33,22 +33,21 @@ services:
       MAXEMAIL_USERNAME: dummyUser
       MAXEMAIL_PASSWORD: dummyPassword
       ELASTIC_APM_DISABLE_SEND: 1
+      DYNAMICS_INSTANCE_URI: https://localhost
+      DYNAMICS_TENANT_ID: xxx
+      DYNAMICS_APPLICATION_ID: xxx
+      DYNAMICS_CLIENT_SECRET: xxx
     ports:
       - 8001:8001
     depends_on:
       - lbdb
-    command:  >
-      bash -c "sleep 5 && python manage.py migrate &&
-      python manage.py collectstatic --noinput &&
-      python -Wd manage.py runserver 0.0.0.0:8001"
-    logging:
-      driver: none
+    command: >
+      bash -c "sleep 5 && python manage.py migrate && python manage.py collectstatic --noinput && python -Wd manage.py runserver 0.0.0.0:8001"
 
   lbdb:
-    image: postgres:9.6.9-alpine
+    image: postgres:16
     restart: unless-stopped
     environment:
       POSTGRES_USER: postgres
       POSTGRES_DB: legal-basis
-    logging:
-      driver: none
+      POSTGRES_PASSWORD: password

manifest.yml

@@ -3,7 +3,7 @@ applications:
   - buildpacks:
       - https://github.com/cloudfoundry/nodejs-buildpack.git#v1.8.23
     health-check-type: http
-    health-check-http-endpoint: /healthcheck
+    health-check-http-endpoint: /pingdom
     memory: 5G
     disk_quota: 8G
     stack: cflinuxfs4

package.json

@@ -23,8 +23,8 @@
     "lint:js": "npx eslint --ext .jsx,.js ./test ./src ./assets",
     "lint:js:fix": "npx eslint --ext .jsx,.js ./test ./src ./assets --fix",
     "test": "run-s test:*",
-    "test:unit:files": "f() { NODE_ENV=test API_ROOT=http://test LOG_LEVEL=silent MOCHA_FILE=junit/test-results.xml npx mocha \"${@:-./src/**/*.test.js}\"; }; f",
-    "test:unit": "NODE_ENV=test API_ROOT=http://test MOCHA_FILE=junit/test-results.xml npx mocha ./src/**/*.test.js --exit",
+    "test:unit:files": "f() { NODE_ENV=test API_ROOT=http://test LOG_LEVEL=info MOCHA_FILE=junit/test-results.xml npx mocha \"${@:-./src/**/*.test.js}\"; }; f",
+    "test:unit": "NODE_ENV=test API_ROOT=http://test LOG_LEVEL=info MOCHA_FILE=junit/test-results.xml npx mocha ./src/**/*.test.js --exit",
     "test:functional": "CYPRESS_retries=3 nyc cypress run --browser chrome",
     "test:a11y": "CYPRESS_coverage=false cypress run --config-file cypress.a11y.config.js --browser chrome",
     "test:a11y:watch": "CYPRESS_coverage=false cypress open --config-file cypress.a11y.config.js --e2e --browser chrome",
@@ -211,6 +211,7 @@
     "sharp": "^0.33.4",
     "sinon": "^18.0.0",
     "sinon-chai": "^3.7.0",
+    "std-mocks": "^2.0.0",
     "storybook": "^8.1.11",
     "supertest": "^7.0.0",
     "webpack": "^5.92.0",

sample.env

@@ -44,9 +44,9 @@ ACTIVITY_STREAM_OUTGOING_URL=https://lookInVault.com/
 ACTIVITY_STREAM_OUTGOING_ACCESS_KEY_ID=some-id
 ACTIVITY_STREAM_OUTGOING_SECRET_ACCESS_KEY=some-secret
 ADMIN_OAUTH2_ENABLED=false
-AWS_ACCESS_KEY_ID=foo
-AWS_DEFAULT_REGION=eu-west-2
-AWS_SECRET_ACCESS_KEY=bar
+DEFAULT_BUCKET_AWS_ACCESS_KEY_ID=foo
+DEFAULT_BUCKET_AWS_DEFAULT_REGION=eu-west-2
+DEFAULT_BUCKET_AWS_SECRET_ACCESS_KEY=bar
 # Workaround for Docker/CircleCI compatibility problem with Python 3.8
 COLUMNS=80
 DATA_HUB_FRONTEND_ACCESS_KEY_ID=frontend-key-id