feat:setup celery for local development

.copilot/image_build_run.sh

@@ -6,6 +6,6 @@ set -e
 # Add commands below to run inside the container after all the other buildpacks have been applied
 export BUILD_STEP='True'
 export COPILOT_ENVIRONMENT_NAME='build'
-export DJANGO_SETTINGS_MODULE="config.settings.base"
+export DJANGO_SETTINGS_MODULE="fbr.settings.base"
 
 poetry run python fbr/manage.py collectstatic --noinput

.github/workflows/code_quality.yml

@@ -65,7 +65,7 @@ jobs:
         run: |
           npm install
           npm run build
-          DJANGO_SETTINGS_MODULE=config.settings.local poetry run fbr/manage.py collectstatic --noinput
+          DJANGO_SETTINGS_MODULE=fbr.settings poetry run fbr/manage.py collectstatic --noinput
 
       # poetry run fbr/manage.py makemigrations --check --dry-run
 

.secrets.baseline

@@ -133,9 +133,9 @@
         "filename": "Makefile",
         "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3",
         "is_verified": false,
-        "line_number": 102
+        "line_number": 65
       }
     ]
   },
-  "generated_at": "2024-09-09T12:08:54Z"
+  "generated_at": "2024-12-10T03:50:08Z"
 }

local_deployment/Dockerfile → Dockerfile

@@ -25,18 +25,25 @@ ENV PYTHONUNBUFFERED=1 \
   PYTHONDONTWRITEBYTECODE=1 \
   DEBUG=1 \
   DJANGO_ADMIN=1 \
-  DJANGO_SETTINGS_MODULE=config.settings.local
+  DJANGO_SETTINGS_MODULE=fbr.settings
 
 # Install nodejs
 RUN apt install -y curl && \
   curl -sL https://deb.nodesource.com/setup_20.x | bash - && \
   apt install -y nodejs
 
 WORKDIR /app
-COPY . /app
 
-# Install poetry and project dependencies
-RUN pip install poetry==1.8.3 && \
-  poetry install --without dev
+RUN pip install poetry
+
+# Copy only the requirements.txt into the container
+COPY requirements.txt /app/
+
+# Install the dependencies specified in requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . /app
 
-CMD ["local_deployment/entry.sh"]
+COPY entry.sh /entry.sh
+RUN chmod +x /entry.sh
+ENTRYPOINT ["/entry.sh"]

Makefile

@@ -34,14 +34,14 @@ drop-database: # Delete project's postgres database
 		fi
 
 build: # Build docker containers for local execution
-	docker build --no-cache -f local_deployment/Dockerfile -t local_deployment .
+	docker build --no-cache -f Dockerfile -t local_deployment .
 	docker compose build
 
 collectstatic: # Run Django collectstatic
-	docker compose run --rm web poetry run python fbr/manage.py collectstatic --noinput
+	docker compose run --rm web poetry run python manage.py collectstatic --noinput
 
 admin: # Create a superuser
-	docker compose exec web poetry run python fbr/manage.py createsuperuser --username admin --email admin@localhost
+	docker compose exec web poetry run python manage.py createsuperuser --username admin --email admin@localhost
 
 first-use: # Initialise for local execution
 	@echo "$(COLOUR_GREEN)Preparing for first use$(COLOUR_NONE)"
@@ -62,6 +62,7 @@ first-use: # Initialise for local execution
 	@echo "$(COLOUR_GREEN)Destroy containers with 'make down'$(COLOUR_NONE)"
 
 up: # Build, (re)create and start containers
+	export DATABASE_URL=postgres://postgres:postgres@localhost:5432/fbr
 	docker compose up -d
 	@echo "$(COLOUR_GREEN)Services are up - use 'make logs' to view service logs$(COLOUR_NONE)"
 
@@ -90,27 +91,29 @@ logs: # View container logs
 	docker compose logs -f -t
 
 test: # Run tests
-	pytest fbr/tests --cov-report term
+	pytest app/tests --cov-report term
 
 bdd: # Run BDD tests
-	HEADLESS_MODE=false SLOW_MO_MS=500 behave ./fbr/tests/bdd/features/ --tags=LOCAL
+	HEADLESS_MODE=false SLOW_MO_MS=500 behave ./app/tests/bdd/features/ --tags=LOCAL
 
 django-shell: # Run a Django shell (on  container)
-	docker compose run web poetry run python fbr/manage.py shell
+	docker compose run web poetry run python manage.py shell
 
 django-shell-local: # Run a Django shell (local django instance)
 	DATABASE_URL=postgres://postgres:postgres@localhost:5432/fbr \
 		DEBUG=True \
 		DJANGO_ADMIN=False \
 		DJANGO_SECRET_KEY=walls-have-ears \
-		DJANGO_SETTINGS_MODULE=config.settings.local \
-		poetry run python fbr/manage.py shell
+		DJANGO_SETTINGS_MODULE=fbr.settings \
+		poetry run python manage.py shell
 
 migrate: # Run Django migrate
-	docker compose run --rm web poetry run python fbr/manage.py migrate --noinput
+	export DATABASE_URL=postgres://postgres:postgres@localhost:5432/fbr && \
+	python manage.py migrate
 
 migrations: # Run Django makemigrations
-	docker compose run --rm web poetry run python fbr/manage.py makemigrations --noinput
+	export DATABASE_URL=postgres://postgres:postgres@localhost:5432/fbr && \
+	python manage.py makemigrations
 
 lint: # Run all linting
 	make black
@@ -126,8 +129,8 @@ secrets-baseline: # Generate a new secrets baseline file
 	poetry run detect-secrets scan > .secrets.baseline
 
 rebuild_cache:
-	export PYTHONPATH=./fbr && \
-	export DJANGO_SETTINGS_MODULE='fbr.config.settings.local' && \
+	export PYTHONPATH=. && \
+	export DJANGO_SETTINGS_MODULE='fbr.settings' && \
 	export DATABASE_URL=postgres://postgres:postgres@localhost:5432/fbr && \
 	poetry install && \
 	poetry run rebuild-cache
@@ -136,6 +139,7 @@ setup_local: # Set up the local environment
 	@echo "$(COLOUR_GREEN)Running initial setup for local environment...$(COLOUR_NONE)"
 	$(MAKE) first-use
 	$(MAKE) start
+	#$(MAKE) migrations
 	$(MAKE) migrate
-	$(MAKE) rebuild_cache
+	#$(MAKE) rebuild_cache
 	@echo "$(COLOUR_GREEN)Local setup complete.$(COLOUR_NONE)"

Procfile

@@ -1,3 +1,4 @@
 web: bash paas_entrypoint.sh
-celery-worker: celery --app fbr.config.celery worker --task-events --loglevel INFO
-celery-beat: celery --app fbr.config.celery beat --loglevel INFO
+celery-worker: celery --app fbr.celery_app worker --task-events --loglevel INFO
+celery-beat: celery --app fbr.celery_app beat --loglevel INFO
+check: python manage.py check

fbr/cache/legislation.py → app/cache/legislation.py

@@ -10,10 +10,10 @@
 
 import requests  # type: ignore
 
-from fbr.cache.construction_legislation import construction_legislation_dataframe
-from fbr.search.config import SearchDocumentConfig
-from fbr.search.utils.date import convert_date_string_to_obj
-from fbr.search.utils.documents import (  # noqa: E501
+from app.cache.construction_legislation import construction_legislation_dataframe
+from app.search.config import SearchDocumentConfig
+from app.search.utils.date import convert_date_string_to_obj
+from app.search.utils.documents import (  # noqa: E501
     generate_short_uuid,
     insert_or_update_document,
 )

fbr/cache/manage_cache.py → app/cache/manage_cache.py

@@ -3,16 +3,17 @@
 
 import django
 
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local")
-django.setup()
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "fbr.settings")
 
+# Initialize Django setup
+django.setup()
 
 import time
 
-from fbr.cache.legislation import Legislation
-from fbr.cache.public_gateway import PublicGateway
-from fbr.search.config import SearchDocumentConfig
-from fbr.search.utils.documents import clear_all_documents
+from app.cache.legislation import Legislation
+from app.cache.public_gateway import PublicGateway
+from app.search.config import SearchDocumentConfig
+from app.search.utils.documents import clear_all_documents
 
 
 def rebuild_cache():

fbr/cache/public_gateway.py → app/cache/public_gateway.py

@@ -4,8 +4,8 @@
 
 import requests  # type: ignore
 
-from fbr.search.utils.date import convert_date_string_to_obj
-from fbr.search.utils.documents import (  # noqa: E501
+from app.search.utils.date import convert_date_string_to_obj
+from app.search.utils.documents import (  # noqa: E501
     generate_short_uuid,
     insert_or_update_document,
 )

fbr/core/apps.py → app/core/apps.py

@@ -2,6 +2,6 @@
 
 
 class CoreConfig(AppConfig):
-    name = "core"
+    name = "app.core"
     verbose_name = "Find business regulations core application functionality"
     default_auto_field = "django.db.models.BigAutoField"

fbr/search/apps.py → app/search/apps.py

@@ -13,6 +13,6 @@ class SearchConfig(AppConfig):
 
     """
 
-    name = "search"
+    name = "app.search"
     verbose_name = "Find business regulations application functionality"
     default_auto_field = "django.db.models.BigAutoField"

fbr/search/utils/documents.py → app/search/utils/documents.py

@@ -6,7 +6,7 @@
 
 from django.db.models import QuerySet
 
-from search.models import DataResponseModel, logger
+from app.search.models import DataResponseModel, logger
 
 
 def clear_all_documents():

fbr/search/utils/paginate.py → app/search/utils/paginate.py

@@ -4,7 +4,7 @@
 from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator
 from django.db.models import QuerySet
 
-from search.config import SearchDocumentConfig
+from app.search.config import SearchDocumentConfig
 
 logger = logging.getLogger(__name__)
 

fbr/search/utils/search.py → app/search/utils/search.py

@@ -6,11 +6,11 @@
 from django.db.models import F, Func, Q, QuerySet
 from django.http import HttpRequest
 
-from search.config import SearchDocumentConfig
-from search.models import DataResponseModel
-from search.utils.documents import calculate_score
-from search.utils.paginate import paginate
-from search.utils.terms import sanitize_input
+from app.search.config import SearchDocumentConfig
+from app.search.models import DataResponseModel
+from app.search.utils.documents import calculate_score
+from app.search.utils.paginate import paginate
+from app.search.utils.terms import sanitize_input
 
 logger = logging.getLogger(__name__)
 
@@ -145,7 +145,9 @@ def search_database(
     return queryset
 
 
-def search(context: dict, request: HttpRequest) -> dict:
+def search(
+    context: dict, request: HttpRequest
+) -> dict | QuerySet[DataResponseModel]:
     logger.debug("received search request: %s", request)
     start_time = time.time()
 
@@ -174,6 +176,9 @@ def search(context: dict, request: HttpRequest) -> dict:
     # Search across specific fields
     results = search_database(config)
 
+    if config.limit == "*":
+        return results
+
     # convert search_results into json
     pag_start_time = time.time()
     context = paginate(context, config, results)

app/search/views.py

@@ -0,0 +1,74 @@
+import logging
+
+from django.conf import settings
+from django.http import HttpRequest, HttpResponse
+from django.shortcuts import render
+from django.views.decorators.http import require_http_methods
+
+from app.search.config import SearchDocumentConfig
+from app.search.utils.search import search, search_database
+
+logger = logging.getLogger(__name__)
+
+
+@require_http_methods(["GET"])
+def document(request: HttpRequest, id) -> HttpResponse:
+    """
+    Document details view.
+
+    Handles the GET request to fetch details based on the provided id.
+    """
+    context = {
+        "service_name": settings.SERVICE_NAME_SEARCH,
+    }
+
+    if not id:
+        context["error"] = "id parameter is required"
+        return render(request, template_name="document.html", context=context)
+
+    # Create a search configuration object with the provided id
+    config = SearchDocumentConfig(search_query="", id=id)
+
+    try:
+        queryset = search_database(config)
+        context["result"] = queryset
+    except Exception as e:
+        logger.error("error fetching details: %s", e)
+        context["error"] = f"error fetching details: {e}"
+
+    return render(request, template_name="document.html", context=context)
+
+
+@require_http_methods(["GET"])
+def search_django(request: HttpRequest):
+    """
+    Search view.
+
+    Renders the Django based search page.
+    """
+    context = {
+        "service_name": settings.SERVICE_NAME_SEARCH,
+    }
+
+    context = search(context, request)
+    return render(request, template_name="django-fbr.html", context=context)
+
+
+@require_http_methods(["GET"])
+def search_react(request: HttpRequest) -> HttpResponse:
+    """
+    Search view.
+
+    Renders the React based search page.
+    """
+
+    context = {
+        "service_name": settings.SERVICE_NAME_SEARCH,
+        "document_types": {
+            "standard": "Standard",
+            "guidance": "Guidance",
+            "legislation": "Legislation",
+        },
+    }
+
+    return render(request, template_name="react-fbr.html", context=context)

fbr/cache/tasks.py → celery_worker/tasks.py

@@ -1,14 +1,15 @@
 import time
 
-from config.celery import celery_app
-from fbr.cache.legislation import Legislation
-from fbr.cache.public_gateway import PublicGateway
-from fbr.search.config import SearchDocumentConfig
-from fbr.search.utils.documents import clear_all_documents
+from celery import shared_task
 
+from app.cache.legislation import Legislation
+from app.cache.public_gateway import PublicGateway
+from app.search.config import SearchDocumentConfig
+from app.search.utils.documents import clear_all_documents
 
-@celery_app.task(name="fbr.cache.tasks.rebuild_cache")
-def rebuild_cache() -> None:
+
+@shared_task(name="celery_worker.tasks.rebuild_cache")
+def rebuild_cache():
     """
     Rebuilds the cache for search documents across various components by
     clearing all existing documents. The process is timed, and the