refactor:restructure project architecture

Reorganized application package structure to improve modularity and navigability. Moved files from `fbr` to `app` directory and renamed key modules to better reflect functionality, such as `celery_worker`. Additionally, updated all relevant imports and settings configurations to align with the new structure, including Django settings and task management.

.copilot/image_build_run.sh

@@ -6,6 +6,6 @@ set -e
 # Add commands below to run inside the container after all the other buildpacks have been applied
 export BUILD_STEP='True'
 export COPILOT_ENVIRONMENT_NAME='build'
-export DJANGO_SETTINGS_MODULE="config.settings.base"
+export DJANGO_SETTINGS_MODULE="fbr.settings.base"
 
 poetry run python fbr/manage.py collectstatic --noinput

.github/workflows/code_quality.yml

@@ -65,7 +65,7 @@ jobs:
         run: |
           npm install
           npm run build
-          DJANGO_SETTINGS_MODULE=config.settings.local poetry run fbr/manage.py collectstatic --noinput
+          DJANGO_SETTINGS_MODULE=fbr.settings poetry run fbr/manage.py collectstatic --noinput
 
       # poetry run fbr/manage.py makemigrations --check --dry-run
 

Makefile

@@ -38,10 +38,10 @@ build: # Build docker containers for local execution
 	docker compose build
 
 collectstatic: # Run Django collectstatic
-	docker compose run --rm web poetry run python fbr/manage.py collectstatic --noinput
+	docker compose run --rm web poetry run python manage.py collectstatic --noinput
 
 admin: # Create a superuser
-	docker compose exec web poetry run python fbr/manage.py createsuperuser --username admin --email admin@localhost
+	docker compose exec web poetry run python manage.py createsuperuser --username admin --email admin@localhost
 
 first-use: # Initialise for local execution
 	@echo "$(COLOUR_GREEN)Preparing for first use$(COLOUR_NONE)"
@@ -90,27 +90,27 @@ logs: # View container logs
 	docker compose logs -f -t
 
 test: # Run tests
-	pytest fbr/tests --cov-report term
+	pytest app/tests --cov-report term
 
 bdd: # Run BDD tests
-	HEADLESS_MODE=false SLOW_MO_MS=500 behave ./fbr/tests/bdd/features/ --tags=LOCAL
+	HEADLESS_MODE=false SLOW_MO_MS=500 behave ./app/tests/bdd/features/ --tags=LOCAL
 
 django-shell: # Run a Django shell (on  container)
-	docker compose run web poetry run python fbr/manage.py shell
+	docker compose run web poetry run python manage.py shell
 
 django-shell-local: # Run a Django shell (local django instance)
 	DATABASE_URL=postgres://postgres:postgres@localhost:5432/fbr \
 		DEBUG=True \
 		DJANGO_ADMIN=False \
 		DJANGO_SECRET_KEY=walls-have-ears \
-		DJANGO_SETTINGS_MODULE=config.settings.local \
-		poetry run python fbr/manage.py shell
+		DJANGO_SETTINGS_MODULE=fbr.settings \
+		poetry run python manage.py shell
 
 migrate: # Run Django migrate
-	docker compose run --rm web poetry run python fbr/manage.py migrate --noinput
+	docker compose run --rm web poetry run python manage.py migrate --noinput
 
 migrations: # Run Django makemigrations
-	docker compose run --rm web poetry run python fbr/manage.py makemigrations --noinput
+	docker compose run --rm web poetry run python manage.py makemigrations --noinput
 
 lint: # Run all linting
 	make black
@@ -126,8 +126,8 @@ secrets-baseline: # Generate a new secrets baseline file
 	poetry run detect-secrets scan > .secrets.baseline
 
 rebuild_cache:
-	export PYTHONPATH=./fbr && \
-	export DJANGO_SETTINGS_MODULE='fbr.config.settings.local' && \
+	export PYTHONPATH=. && \
+	export DJANGO_SETTINGS_MODULE='fbr.settings' && \
 	export DATABASE_URL=postgres://postgres:postgres@localhost:5432/fbr && \
 	poetry install && \
 	poetry run rebuild-cache

Procfile

@@ -1,3 +1,4 @@
 web: bash paas_entrypoint.sh
-celery-worker: celery --app fbr.config.celery worker --task-events --loglevel INFO
-celery-beat: celery --app fbr.config.celery beat --loglevel INFO
+celery-worker: celery --app fbr.celery_app worker --task-events --loglevel INFO
+celery-beat: celery --app fbr.celery_app beat --loglevel INFO
+check: python manage.py check

fbr/cache/legislation.py → app/cache/legislation.py

@@ -10,10 +10,10 @@
 
 import requests  # type: ignore
 
-from fbr.cache.construction_legislation import construction_legislation_dataframe
-from fbr.search.config import SearchDocumentConfig
-from fbr.search.utils.date import convert_date_string_to_obj
-from fbr.search.utils.documents import (  # noqa: E501
+from app.cache.construction_legislation import construction_legislation_dataframe
+from app.search.config import SearchDocumentConfig
+from app.search.utils.date import convert_date_string_to_obj
+from app.search.utils.documents import (  # noqa: E501
     generate_short_uuid,
     insert_or_update_document,
 )

fbr/cache/manage_cache.py → app/cache/manage_cache.py

@@ -3,16 +3,17 @@
 
 import django
 
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local")
-django.setup()
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "fbr.settings")
 
+# Initialize Django setup
+django.setup()
 
 import time
 
-from fbr.cache.legislation import Legislation
-from fbr.cache.public_gateway import PublicGateway
-from fbr.search.config import SearchDocumentConfig
-from fbr.search.utils.documents import clear_all_documents
+from app.cache.legislation import Legislation
+from app.cache.public_gateway import PublicGateway
+from app.search.config import SearchDocumentConfig
+from app.search.utils.documents import clear_all_documents
 
 
 def rebuild_cache():

fbr/cache/public_gateway.py → app/cache/public_gateway.py

@@ -4,8 +4,8 @@
 
 import requests  # type: ignore
 
-from fbr.search.utils.date import convert_date_string_to_obj
-from fbr.search.utils.documents import (  # noqa: E501
+from app.search.utils.date import convert_date_string_to_obj
+from app.search.utils.documents import (  # noqa: E501
     generate_short_uuid,
     insert_or_update_document,
 )

fbr/core/apps.py → app/core/apps.py

@@ -2,6 +2,6 @@
 
 
 class CoreConfig(AppConfig):
-    name = "core"
+    name = "app.core"
     verbose_name = "Find business regulations core application functionality"
     default_auto_field = "django.db.models.BigAutoField"

fbr/search/apps.py → app/search/apps.py

@@ -13,6 +13,6 @@ class SearchConfig(AppConfig):
 
     """
 
-    name = "search"
+    name = "app.search"
     verbose_name = "Find business regulations application functionality"
     default_auto_field = "django.db.models.BigAutoField"

fbr/search/utils/documents.py → app/search/utils/documents.py

@@ -6,7 +6,7 @@
 
 from django.db.models import QuerySet
 
-from search.models import DataResponseModel, logger
+from app.search.models import DataResponseModel, logger
 
 
 def clear_all_documents():

fbr/search/utils/paginate.py → app/search/utils/paginate.py

@@ -4,7 +4,7 @@
 from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator
 from django.db.models import QuerySet
 
-from search.config import SearchDocumentConfig
+from app.search.config import SearchDocumentConfig
 
 logger = logging.getLogger(__name__)
 

fbr/search/utils/search.py → app/search/utils/search.py

@@ -6,11 +6,11 @@
 from django.db.models import F, Func, Q, QuerySet
 from django.http import HttpRequest
 
-from search.config import SearchDocumentConfig
-from search.models import DataResponseModel
-from search.utils.documents import calculate_score
-from search.utils.paginate import paginate
-from search.utils.terms import sanitize_input
+from app.search.config import SearchDocumentConfig
+from app.search.models import DataResponseModel
+from app.search.utils.documents import calculate_score
+from app.search.utils.paginate import paginate
+from app.search.utils.terms import sanitize_input
 
 logger = logging.getLogger(__name__)
 
@@ -145,7 +145,9 @@ def search_database(
     return queryset
 
 
-def search(context: dict, request: HttpRequest) -> dict:
+def search(
+    context: dict, request: HttpRequest
+) -> dict | QuerySet[DataResponseModel]:
     logger.debug("received search request: %s", request)
     start_time = time.time()
 
@@ -174,6 +176,9 @@ def search(context: dict, request: HttpRequest) -> dict:
     # Search across specific fields
     results = search_database(config)
 
+    if config.limit == "*":
+        return results
+
     # convert search_results into json
     pag_start_time = time.time()
     context = paginate(context, config, results)

app/search/views.py

@@ -0,0 +1,74 @@
+import logging
+
+from django.conf import settings
+from django.http import HttpRequest, HttpResponse
+from django.shortcuts import render
+from django.views.decorators.http import require_http_methods
+
+from app.search.config import SearchDocumentConfig
+from app.search.utils.search import search, search_database
+
+logger = logging.getLogger(__name__)
+
+
+@require_http_methods(["GET"])
+def document(request: HttpRequest, id) -> HttpResponse:
+    """
+    Document details view.
+
+    Handles the GET request to fetch details based on the provided id.
+    """
+    context = {
+        "service_name": settings.SERVICE_NAME_SEARCH,
+    }
+
+    if not id:
+        context["error"] = "id parameter is required"
+        return render(request, template_name="document.html", context=context)
+
+    # Create a search configuration object with the provided id
+    config = SearchDocumentConfig(search_query="", id=id)
+
+    try:
+        queryset = search_database(config)
+        context["result"] = queryset
+    except Exception as e:
+        logger.error("error fetching details: %s", e)
+        context["error"] = f"error fetching details: {e}"
+
+    return render(request, template_name="document.html", context=context)
+
+
+@require_http_methods(["GET"])
+def search_django(request: HttpRequest):
+    """
+    Search view.
+
+    Renders the Django based search page.
+    """
+    context = {
+        "service_name": settings.SERVICE_NAME_SEARCH,
+    }
+
+    context = search(context, request)
+    return render(request, template_name="django-fbr.html", context=context)
+
+
+@require_http_methods(["GET"])
+def search_react(request: HttpRequest) -> HttpResponse:
+    """
+    Search view.
+
+    Renders the React based search page.
+    """
+
+    context = {
+        "service_name": settings.SERVICE_NAME_SEARCH,
+        "document_types": {
+            "standard": "Standard",
+            "guidance": "Guidance",
+            "legislation": "Legislation",
+        },
+    }
+
+    return render(request, template_name="react-fbr.html", context=context)

fbr/cache/tasks.py → celery_worker/tasks.py

@@ -1,13 +1,14 @@
 import time
 
-from config.celery import celery_app
-from fbr.cache.legislation import Legislation
-from fbr.cache.public_gateway import PublicGateway
-from fbr.search.config import SearchDocumentConfig
-from fbr.search.utils.documents import clear_all_documents
+from celery import shared_task
 
+from app.cache.legislation import Legislation
+from app.cache.public_gateway import PublicGateway
+from app.search.config import SearchDocumentConfig
+from app.search.utils.documents import clear_all_documents
 
-@celery_app.task(name="fbr.cache.tasks.rebuild_cache")
+
+@shared_task(bind=True)
 def rebuild_cache() -> None:
     """
     Rebuilds the cache for search documents across various components by

docker-compose.yml

@@ -2,6 +2,8 @@
 volumes:
   postgres_data:
     driver: local
+  redis_data:
+      driver: local
 
 services:
   db:
@@ -21,6 +23,7 @@ services:
       retries: 3
     networks:
       - proxynet
+
   web:
     build:
       context: .
@@ -37,6 +40,58 @@ services:
     networks:
       - proxynet
 
+  redis:
+    image: redis
+    # Expose port so we can query it for debugging
+    ports:
+      - "6379:6379"
+
+  celery-worker:
+    build:
+      context: .
+      dockerfile: local_deployment/Dockerfile
+    command: celery --app fbr.celery_app worker --task-events --loglevel INFO
+    entrypoint: ''
+    volumes:
+      - .:/app
+    healthcheck:
+      test: [ "CMD-SHELL", "python -m dbt_copilot_python.celery_health_check.healthcheck" ]
+      interval: 10s
+      timeout: 5s
+      retries: 2
+      start_period: 5s
+    depends_on:
+      - redis
+      - db
+    environment:
+      REDIS_ENDPOINT: redis://redis:6379
+      DEBUG: true
+      DJANGO_SETTINGS_MODULE: fbr.settings
+      RDS_POSTGRES_CREDENTIALS: '{"password":"postgres","dbname":"fbr","engine":"postgres","port":5432,"dbInstanceIdentifier":"xxx","host":"db","username":"postgres"}'
+
+  celery-beats:
+    build:
+      context: .
+      dockerfile: local_deployment/Dockerfile
+    command: celery --app fbr.celery_app beat --loglevel INFO
+    entrypoint: ''
+    volumes:
+      - .:/app
+    healthcheck:
+      test: [ "CMD-SHELL", "python -m dbt_copilot_python.celery_health_check.healthcheck" ]
+      interval: 10s
+      timeout: 5s
+      retries: 2
+      start_period: 5s
+    depends_on:
+      - redis
+      - db
+    environment:
+      REDIS_ENDPOINT: redis://redis:6379
+      DEBUG: true
+      DJANGO_SETTINGS_MODULE: fbr.settings
+      RDS_POSTGRES_CREDENTIALS: '{"password":"postgres","dbname":"fbr","engine":"postgres","port":5432,"dbInstanceIdentifier":"xxx","host":"db","username":"postgres"}'
+
 networks:
   proxynet:
     name: fbr_network

fbr/__init__.py

@@ -1,7 +1,3 @@
-from __future__ import absolute_import, unicode_literals
-
-# This ensures the app is always imported when Django starts
-# allowing shared_task to utilize this app.
-from .config.celery import celery_app
+from .celery_app import celery_app
 
 __all__ = ("celery_app",)

fbr/config/settings/base.py

@@ -54,6 +54,7 @@
     "django.contrib.sessions",
     "django.contrib.messages",
     "django.contrib.staticfiles",
+    "rest_framework",
 ]
 
 LOCAL_APPS = [