Merge branch 'dev' into LTD-improve-validation-during-submission

.InstallPackages

@@ -0,0 +1,13 @@
+libpq-dev
+build-essential
+libcairo2
+libpango-1.0-0
+libpangocairo-1.0-0
+libgdk-pixbuf2.0-0
+libffi-dev
+shared-mime-info
+swig
+imagemagick
+poppler-utils
+libsqlite3-dev
+

.circleci/config.yml

@@ -62,7 +62,7 @@ commands:
             sudo apt-get update
             sudo apt-get install swig libpango1.0-0 libcairo2 libpq-dev libpangocairo-1.0-0 imagemagick poppler-utils
 
-  setup:
+  git_checkout:
     steps:
       - checkout
       - run:
@@ -89,6 +89,9 @@ commands:
                   git checkout << pipeline.parameters.lite_routing_branch_name >>
                   cd ..
 
+  setup:
+    steps:
+      - git_checkout
       # Download and cache dependencies
       # ensure this step occurs *before* installing dependencies
       - restore_cache:
@@ -134,7 +137,7 @@ jobs:
       - run:
           name: Run tests
           command: |
-            pipenv run pytest --circleci-parallelize --cov=. --cov-report xml --cov-config=.coveragerc --ignore lite_routing --ignore api/anonymised_db_dumps -k "not seeding and not elasticsearch and not performance and not migration and not db_anonymiser"
+            pipenv run pytest --circleci-parallelize --cov=. --cov-report xml --cov-config=.coveragerc --ignore lite_routing --ignore api/anonymised_db_dumps -k "not seeding and not elasticsearch and not performance and not migration and not db_anonymiser and not requires_transactions"
       - upload_code_coverage:
           alias: tests
 
@@ -154,7 +157,7 @@ jobs:
       - run:
           name: Run tests on Postgres 13
           command: |
-            pipenv run pytest --circleci-parallelize --cov=. --cov-report xml --cov-config=.coveragerc --ignore lite_routing --ignore api/anonymised_db_dumps -k "not seeding and not elasticsearch and not performance and not migration and not db_anonymiser"
+            pipenv run pytest --circleci-parallelize --cov=. --cov-report xml --cov-config=.coveragerc --ignore lite_routing --ignore api/anonymised_db_dumps -k "not seeding and not elasticsearch and not performance and not migration and not db_anonymiser and not requires_transactions"
       - upload_code_coverage:
           alias: tests_dbt_platform
 
@@ -232,6 +235,44 @@ jobs:
       - upload_code_coverage:
           alias: anonymised_db_dump_tests_dbt_platform
 
+  requires_transactions_tests:
+    docker:
+      - <<: *image_python
+      - <<: *image_postgres
+      - <<: *image_opensearch_v1
+      - <<: *image_redis
+    working_directory: ~/lite-api
+    environment:
+      <<: *common_env_vars
+      LITE_API_ENABLE_ES: True
+    steps:
+      - setup
+      - run:
+          name: Run requiring transactions tests
+          command: |
+            pipenv run pytest --cov=. --cov-report xml --cov-config=.coveragerc -k requires_transactions
+      - upload_code_coverage:
+          alias: requires_transactions_tests
+
+  requires_transactions_tests_dbt_platform:
+    docker:
+      - <<: *image_python
+      - <<: *image_postgres13
+      - <<: *image_opensearch
+      - <<: *image_redis
+    working_directory: ~/lite-api
+    environment:
+      <<: *common_env_vars
+      LITE_API_ENABLE_ES: True
+    steps:
+      - setup
+      - run:
+          name: Run requiring transactions tests on Postgres 13
+          command: |
+            pipenv run pytest --cov=. --cov-report xml --cov-config=.coveragerc -k requires_transactions
+      - upload_code_coverage:
+          alias: requires_transactions_tests_dbt_platform
+
   migration_tests:
     docker:
       - <<: *image_python
@@ -242,6 +283,7 @@ jobs:
     environment:
       <<: *common_env_vars
       LITE_API_ENABLE_ES: True
+    parallelism: 5
     steps:
       - setup
       - run:
@@ -261,6 +303,7 @@ jobs:
     environment:
       <<: *common_env_vars
       LITE_API_ENABLE_ES: True
+    parallelism: 5
     steps:
       - setup
       - run:
@@ -436,16 +479,48 @@ jobs:
           name: Check migrations are made
           command: pipenv run ./manage.py makemigrations --check
 
+  check_routing_docs:
+    docker:
+      - <<: *image_python
+      - <<: *image_postgres
+      - <<: *image_opensearch_v1
+    working_directory: ~/lite-api
+    environment:
+      <<: *common_env_vars
+      LITE_API_ENABLE_ES: True
+    steps:
+      - setup
+      - run:
+          name: Migrate app
+          command: pipenv run ./manage.py migrate
+      - run:
+          name: Seed app
+          command: pipenv run ./manage.py seedall
+      - run:
+          name: Generate routing rules documentation
+          command: pipenv run ./manage.py generate_rules_docs
+      - run:
+          name: Check for routing documentation changes
+          command: |
+            cd lite_routing
+            CHANGED_FILES=$(git diff --name-only)
+            CHANGED_CONTENT=$(git diff)
+            if [[ "$CHANGED_FILES" == *"docs/"* ]]; then
+              echo "lite-routing docs are stale! They should be re-generated with manage.py generate_rules_docs and committed to the lite-routing repository."
+              echo "Changed files; $CHANGED_FILES"
+              echo "$CHANGED_CONTENT"
+              exit 1
+            fi
+
   check_coverage:
     working_directory: ~/lite-api
     docker:
       - <<: *image_python
     steps:
-      - checkout
-      - run: git submodule sync --recursive && git submodule update --recursive --init
+      - git_checkout
       - attach_workspace:
           at: ~/lite-api/tmp
-      - run: pip install coverage diff_cover
+      - run: pip install coverage==7.6.4 diff_cover
       - run: coverage combine tmp
       - run: coverage xml
       - run: coverage html
@@ -613,7 +688,11 @@ workflows:
               - open_search_tests
               - migration_tests
               - lite_routing_tests
+              - requires_transactions_tests
       - check-lite-routing-sha
+      - check_routing_docs
       - e2e_tests
       - anonymised_db_dump_tests
       - anonymised_db_dump_tests_dbt_platform
+      - requires_transactions_tests
+      - requires_transactions_tests_dbt_platform

.copilot/config.yml

@@ -0,0 +1,7 @@
+repository: lite/lite-backend
+builder:
+  name: paketobuildpacks/builder-jammy-full
+  version: 0.3.339
+packs:
+  - acodeninja/install
+  - acodeninja/psql

.copilot/image_build_run.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+set -e

.copilot/phases/build.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+set -e

.copilot/phases/install.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+set -e

.copilot/phases/post_build.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+set -e

.copilot/phases/pre_build.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+# Exit early if something goes wrong
+set -e
+
+git_clone_base_url="https://codestar-connections.eu-west-2.amazonaws.com/git-http/$AWS_ACCOUNT_ID/eu-west-2/$CODESTAR_CONNECTION_ID/uktrade"
+
+git config --global credential.helper '!aws codecommit credential-helper $@'
+git config --global credential.UseHttpPath true
+
+cat <<EOF > ./.gitmodules
+[submodule "lite-content"]
+path = lite_content
+url = $git_clone_base_url/lite-content.git
+branch = master
+[submodule "lite_routing"]
+path = lite_routing
+url = $git_clone_base_url/lite-routing.git
+branch = main
+[submodule "django_db_anonymiser"]
+path = django_db_anonymiser
+url = $git_clone_base_url/django-db-anonymiser.git
+EOF
+
+git submodule update --init --recursive

.coveragerc

@@ -6,7 +6,9 @@ omit =
     ./*/__init__.py
     ./*/models.py
     ./*/migrations/*
+    ./api/conf/schema_generator_urls.py
     ./api/conf/settings.py
+    ./api/conf/gconfig-dbt-platform.py
     ./api/conf/wsgi.py
     ./static/management/*
     *test*

.gitignore

@@ -171,7 +171,6 @@ bank-holidays.csv
 /test_helpers/test_endpoints/results/*.csv
 *.p12
 
-.python-version
 
 # pii file
 .pii-secret-hook

Pipfile

@@ -15,12 +15,14 @@ requests-mock = "~=1.8"
 freezegun = "~=1.0.0"
 pytest = "*"
 pytest-django = "*"
-pytest-bdd = "*"
+pytest-bdd = "~=7.3.0"
 ipdb = "*"
 watchdog = {extras = ["watchmedo"], version = "*"}
 diff-pdf-visually = "~=1.7.0"
 pytest-circleci-parallelized = "~=0.1.0"
 moto = {extras = ["s3"], version = "==5.0.3"}
+drf-spectacular = "*"
+pytest-mock = "~=3.14.0"
 
 [packages]
 factory-boy = "~=2.12.0"
@@ -29,7 +31,7 @@ faker = "~=23.2.1"
 boto3 = "~=1.26.17"
 django-activity-stream = "~=0.10.0"
 django-allow-cidr = "~=0.5.0"
-django-elasticsearch-dsl = "~=7.2.2"
+django-elasticsearch-dsl = "~=7.4.0"
 django-elasticsearch-dsl-drf = "~=0.22.5"
 django-environ = "~=0.9.0"
 django-health-check = "~=3.18.1"
@@ -48,15 +50,14 @@ defusedxml = "~=0.6.0"
 xmlschema = "~=1.2.0"
 notifications-python-client = "~=5.7.1"
 dataclasses = "~=0.6"
-# endesive = "~=1.5.9" (not included here as installed in a later jenkins step. Uncomment if working locally)
 pypdf2 = "~=1.27.5"
-cryptography = "~=42.0.0"
-sentry-sdk = "~=1.17.0"
+cryptography = "~=43.0.1"
+sentry-sdk = "~=2.8.0"
 elastic-apm = "~=6.7.2"
 gunicorn = "~=22.0.0"
 gevent = "~=23.9.1"
 xmltodict = "~=0.12.0"
-Pillow = "~=10.2.0"
+Pillow = "~=10.3.0"
 django-phonenumber-field = "==6.4.0"
 phonenumbers = "~=8.13.1"
 django-log-formatter-ecs = "==0.0.5"
@@ -68,14 +69,18 @@ celery = "~=5.3.0"
 redis = "~=4.4.4"
 django-test-migrations = "~=1.2.0"
 django-silk = "~=5.0.3"
-django = "~=4.2.10"
+django = "~=4.2.15"
 django-queryable-properties = "~=1.9.1"
 database-sanitizer = ">=1.1.0"
 django-reversion = ">=5.0.12"
 psycopg = "~=3.1.18"
 django-log-formatter-asim = "~=0.0.5"
 dbt-copilot-python = "~=0.2.1"
 dj-database-url = "~=2.2.0"
+certifi = "~=2024.7.4"
+pytz = "~=2024.1"
+drf-spectacular = "~=0.27.2"
+djangorestframework-csv = "~=3.0.2"
 
 [requires]
 python_version = "3.9"