Merge branch 'dev' into LTD-improve-validation-during-submission

.circleci/config.yml

@@ -8,26 +8,42 @@ common_env_vars: &common_env_vars
   AV_SERVICE_PASSWORD: AV_SERVICE_PASSWORD
 
 image_python: &image_python
-  image: cimg/python:3.8.18
+  image: cimg/python:3.9.18
 
 image_python_node: &image_python_node
-  image: cimg/python:3.8.18-node
+  image: cimg/python:3.9.18-node
 
 image_postgres: &image_postgres
   image: circleci/postgres:12
   environment:
     POSTGRES_USER: postgres
     POSTGRES_PASSWORD: password
 
-image_elasticsearch: &image_elasticsearch
-  image: docker.elastic.co/elasticsearch/elasticsearch:7.10.1
+image_postgres13: &image_postgres13
+  image: circleci/postgres:13
   environment:
-    ES_JAVA_OPTS: "-Xmx256m -Xms256m"
-    xpack.security.enabled: "false"
-    xpack.monitoring.enabled: "false"
-    xpack.graph.enabled: "false"
-    xpack.watcher.enabled: "false"
-    transport.host: "localhost"
+    POSTGRES_USER: postgres
+    POSTGRES_PASSWORD: password
+
+image_opensearch_v1: &image_opensearch_v1
+  image: opensearchproject/opensearch:1.3.13
+  environment:
+    - cluster.name=cluster-001
+    - node.name=node-001
+    - discovery.type=single-node
+    - bootstrap.memory_lock=true
+    - DISABLE_INSTALL_DEMO_CONFIG=true # Prevents execution of bundled demo script which installs demo certificates and security configurations to OpenSearch
+    - DISABLE_SECURITY_PLUGIN=true # Disables security plugin
+
+image_opensearch: &image_opensearch
+  image: opensearchproject/opensearch:2.14.0
+  environment:
+    - cluster.name=cluster-001
+    - node.name=node-001
+    - discovery.type=single-node
+    - bootstrap.memory_lock=true
+    - DISABLE_INSTALL_DEMO_CONFIG=true # Prevents execution of bundled demo script which installs demo certificates and security configurations to OpenSearch
+    - DISABLE_SECURITY_PLUGIN=true # Disables security plugin
 
 image_redis: &image_redis
   image: circleci/redis:3.2-alpine
@@ -106,7 +122,7 @@ jobs:
     docker:
       - <<: *image_python
       - <<: *image_postgres
-      - <<: *image_elasticsearch
+      - <<: *image_opensearch_v1
       - <<: *image_redis
     working_directory: ~/lite-api
     environment:
@@ -122,11 +138,31 @@ jobs:
       - upload_code_coverage:
           alias: tests
 
+  tests_dbt_platform:
+    docker:
+      - <<: *image_python
+      - <<: *image_postgres13
+      - <<: *image_opensearch
+      - <<: *image_redis
+    working_directory: ~/lite-api
+    environment:
+      <<: *common_env_vars
+      LITE_API_ENABLE_ES: True
+    parallelism: 10
+    steps:
+      - setup
+      - run:
+          name: Run tests on Postgres 13
+          command: |
+            pipenv run pytest --circleci-parallelize --cov=. --cov-report xml --cov-config=.coveragerc --ignore lite_routing --ignore api/anonymised_db_dumps -k "not seeding and not elasticsearch and not performance and not migration and not db_anonymiser"
+      - upload_code_coverage:
+          alias: tests_dbt_platform
+
   seeding_tests:
     docker:
       - <<: *image_python
       - <<: *image_postgres
-      - <<: *image_elasticsearch
+      - <<: *image_opensearch_v1
     working_directory: ~/lite-api
     environment:
       <<: *common_env_vars
@@ -140,11 +176,29 @@ jobs:
       - upload_code_coverage:
           alias: seeding_tests
 
+  seeding_tests_dbt_platform:
+    docker:
+      - <<: *image_python
+      - <<: *image_postgres13
+      - <<: *image_opensearch
+    working_directory: ~/lite-api
+    environment:
+      <<: *common_env_vars
+      LITE_API_ENABLE_ES: True
+    steps:
+      - setup
+      - run:
+          name: Run seeding tests on Postgres 13
+          command: |
+            pipenv run pytest --cov=. --cov-report xml --cov-config=.coveragerc -k seeding
+      - upload_code_coverage:
+          alias: seeding_tests_dbt_platform
+
   anonymised_db_dump_tests:
     docker:
       - <<: *image_python
       - <<: *image_postgres
-      - <<: *image_elasticsearch
+      - <<: *image_opensearch_v1
       - <<: *image_redis
     working_directory: ~/lite-api
     environment:
@@ -159,11 +213,30 @@ jobs:
       - upload_code_coverage:
           alias: anonymised_db_dumps
 
+  anonymised_db_dump_tests_dbt_platform:
+    docker:
+      - <<: *image_python
+      - <<: *image_postgres13
+      - <<: *image_opensearch
+      - <<: *image_redis
+    working_directory: ~/lite-api
+    environment:
+      <<: *common_env_vars
+      LITE_API_ENABLE_ES: True
+    steps:
+      - setup
+      - run:
+          name: Run anonymised DB dump tests on Postgres 13
+          command: |
+            pipenv run pytest --cov=. --cov-report xml --cov-config=.coveragerc api/anonymised_db_dumps
+      - upload_code_coverage:
+          alias: anonymised_db_dump_tests_dbt_platform
+
   migration_tests:
     docker:
       - <<: *image_python
       - <<: *image_postgres
-      - <<: *image_elasticsearch
+      - <<: *image_opensearch_v1
       - <<: *image_redis
     working_directory: ~/lite-api
     environment:
@@ -178,11 +251,30 @@ jobs:
       - upload_code_coverage:
           alias: migration_tests
 
+  migration_tests_dbt_platform:
+    docker:
+      - <<: *image_python
+      - <<: *image_postgres13
+      - <<: *image_opensearch
+      - <<: *image_redis
+    working_directory: ~/lite-api
+    environment:
+      <<: *common_env_vars
+      LITE_API_ENABLE_ES: True
+    steps:
+      - setup
+      - run:
+          name: Run migration tests for dbt platform
+          command: |
+            pipenv run pytest --circleci-parallelize --cov=. --cov-report xml --cov-config=.coveragerc -k migration
+      - upload_code_coverage:
+          alias: migration_tests_dbt_platform
+
   lite_routing_tests:
     docker:
       - <<: *image_python
       - <<: *image_postgres
-      - <<: *image_elasticsearch
+      - <<: *image_opensearch_v1
       - <<: *image_redis
     working_directory: ~/lite-api
     environment:
@@ -198,11 +290,61 @@ jobs:
       - upload_code_coverage:
           alias: lite_routing_tests
 
+  lite_routing_tests_dbt_platform:
+    docker:
+      - <<: *image_python
+      - <<: *image_postgres13
+      - <<: *image_opensearch
+      - <<: *image_redis
+    working_directory: ~/lite-api
+    environment:
+      <<: *common_env_vars
+      LITE_API_ENABLE_ES: True
+    parallelism: 5
+    steps:
+      - setup
+      - run:
+          name: Run lite_routing tests for dbt platform
+          command: |
+            pipenv run pytest --circleci-parallelize --cov=. --cov-report xml --cov-config=.coveragerc -k "not migration" --ignore lite_routing/routing_rules_internal/tests/bdd lite_routing
+      - upload_code_coverage:
+          alias: lite_routing_tests_dbt_platform
+
   lite_routing_bdd_tests:
     docker:
       - <<: *image_python_node
       - <<: *image_postgres
-      - <<: *image_elasticsearch
+      - <<: *image_opensearch_v1
+      - <<: *image_redis
+    working_directory: ~/lite-api
+    environment:
+      <<: *common_env_vars
+      LITE_API_ENABLE_ES: True
+    parallelism: 5
+    steps:
+      - setup
+      - run:
+          name: Install cucumber reporter package
+          command: npm install multiple-cucumber-html-reporter
+      - run:
+          name: Create report directories
+          command: |
+            mkdir cucumber_results
+      - run:
+          name: Run lite_routing tests
+          command: pipenv run pytest --circleci-parallelize --gherkin-terminal-reporter -vv lite_routing/routing_rules_internal/tests/bdd --cucumberjson=cucumber_results/cuc.json
+      - run:
+          name: Generate html cucumber report
+          command: node generate_cucumber_report.js
+          when: always
+      - store_artifacts:
+          path: cucumber_html
+
+  lite_routing_bdd_tests_dbt_platform:
+    docker:
+      - <<: *image_python_node
+      - <<: *image_postgres13
+      - <<: *image_opensearch
       - <<: *image_redis
     working_directory: ~/lite-api
     environment:
@@ -228,29 +370,62 @@ jobs:
       - store_artifacts:
           path: cucumber_html
 
-  elastic_search_tests:
+  open_search_tests:
     docker:
       - <<: *image_python
       - <<: *image_postgres
-      - <<: *image_elasticsearch
+      - <<: *image_opensearch_v1
+    working_directory: ~/lite-api
+    environment:
+      <<: *common_env_vars
+      LITE_API_ENABLE_ES: True
+    steps:
+      - setup
+      - run:
+          name: Run opensearch tests v1
+          command: |
+            pipenv run pytest --cov=. --cov-report xml --cov-config=.coveragerc -k elasticsearch
+      - upload_code_coverage:
+          alias: open_search_v1_tests
+
+  open_search_tests_dbt_platform:
+    docker:
+      - <<: *image_python
+      - <<: *image_postgres13
+      - <<: *image_opensearch
     working_directory: ~/lite-api
     environment:
       <<: *common_env_vars
       LITE_API_ENABLE_ES: True
     steps:
       - setup
       - run:
-          name: Run elasticsearch tests
+          name: Run opensearch tests on dbt platform config
           command: |
             pipenv run pytest --cov=. --cov-report xml --cov-config=.coveragerc -k elasticsearch
       - upload_code_coverage:
-          alias: elastic_search_tests
+          alias: open_search_tests_dbt_platform
 
   check_migrations:
     docker:
       - <<: *image_python
       - <<: *image_postgres
-      - <<: *image_elasticsearch
+      - <<: *image_opensearch_v1
+    working_directory: ~/lite-api
+    environment:
+      <<: *common_env_vars
+      LITE_API_ENABLE_ES: True
+    steps:
+      - setup
+      - run:
+          name: Check migrations are made
+          command: pipenv run ./manage.py makemigrations --check
+
+  check_migrations_dbt_platform:
+    docker:
+      - <<: *image_python
+      - <<: *image_postgres13
+      - <<: *image_opensearch
     working_directory: ~/lite-api
     environment:
       <<: *common_env_vars
@@ -289,6 +464,10 @@ jobs:
       - <<: *image_python
     steps:
       - setup
+      - run: pipenv install pre-commit
+      - restore_cache:
+          keys:
+            - pre-commit-cache-{{ checksum ".pre-commit-config.yaml" }}
       - run:
           name: Prospector
           command: pipenv run prospector -W pylint -W pep257 -W mccabe
@@ -299,6 +478,13 @@ jobs:
           name: Bandit
           command: |
             pipenv run bandit -r .
+      - run:
+          name: Ruff
+          command: pipenv run pre-commit run ruff --from-ref origin/HEAD --to-ref HEAD
+      - save_cache:
+          key: pre-commit-cache-{{ checksum ".pre-commit-config.yaml" }}
+          paths:
+            - ~/.cache/pre-commit
 
   check-lite-routing-sha:
     docker:
@@ -331,7 +517,7 @@ jobs:
 
   e2e_tests:
     docker:
-      - image: cimg/python:3.8.18
+      - image: cimg/python:3.9.18
     steps:
       - run:
           name: Add known hosts
@@ -409,18 +595,25 @@ workflows:
       - linting
       - tests
       - seeding_tests
+      - seeding_tests_dbt_platform
       - lite_routing_tests
+      - lite_routing_tests_dbt_platform
       - lite_routing_bdd_tests
-      - elastic_search_tests
+      - lite_routing_bdd_tests_dbt_platform
+      - open_search_tests_dbt_platform
+      - open_search_tests
       - migration_tests
+      - migration_tests_dbt_platform
       - check_migrations
+      - check_migrations_dbt_platform
       - check_coverage:
           requires:
               - tests
               - seeding_tests
-              - elastic_search_tests
+              - open_search_tests
               - migration_tests
               - lite_routing_tests
       - check-lite-routing-sha
       - e2e_tests
       - anonymised_db_dump_tests
+      - anonymised_db_dump_tests_dbt_platform

.pre-commit-config.yaml

@@ -1,7 +1,14 @@
 default_language_version:
-    python: python3.8
+    python: python3.9
 
 repos:
+  # ruff-pre-commit docs recommend running ruff before black
+    - repo: https://github.com/astral-sh/ruff-pre-commit
+      rev: v0.1.4
+      hooks:
+        # Config for ruff lives in pyproject.toml
+        - id: ruff
+          args: [ --fix ]
     - repo: https://github.com/psf/black
       rev: 24.3.0
       hooks: