From 0015d5674f00a226b07befaf3145e93ccd6055e7 Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Thu, 28 Nov 2024 17:58:33 +0000 Subject: [PATCH 1/6] feat(orpd-120):implemented cache with cron job The file fbr/search/construction_legislation.py has been deleted because it is no longer in use and its content is obsolete. This cleanup helps streamline the codebase and minimizes potential confusion with outdated legislation references. --- fbr/cache/__init__.py | 0 .../construction_legislation.py | 0 fbr/{search => cache}/legislation.py | 16 ++- fbr/{search => cache}/public_gateway.py | 4 +- fbr/cache/tasks.py | 18 +++ fbr/config/celery.py | 9 ++ fbr/config/settings/base.py | 7 ++ fbr/config/urls.py | 16 +-- .../commands/setup_periodic_task.py | 27 +++++ fbr/search/tests/test_search.py | 24 ++-- fbr/search/views.py | 7 +- poetry.lock | 108 +++++++++++++++++- pyproject.toml | 6 + 13 files changed, 209 insertions(+), 33 deletions(-) create mode 100644 fbr/cache/__init__.py rename fbr/{search => cache}/construction_legislation.py (100%) rename fbr/{search => cache}/legislation.py (96%) rename fbr/{search => cache}/public_gateway.py (96%) create mode 100644 fbr/cache/tasks.py create mode 100644 fbr/config/celery.py create mode 100644 fbr/management/commands/setup_periodic_task.py diff --git a/fbr/cache/__init__.py b/fbr/cache/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fbr/search/construction_legislation.py b/fbr/cache/construction_legislation.py similarity index 100% rename from fbr/search/construction_legislation.py rename to fbr/cache/construction_legislation.py diff --git a/fbr/search/legislation.py b/fbr/cache/legislation.py similarity index 96% rename from fbr/search/legislation.py rename to fbr/cache/legislation.py index c0005e6..bbcefc8 100644 --- a/fbr/search/legislation.py +++ b/fbr/cache/legislation.py @@ -1,3 +1,6 @@ +# isort: skip_file +# fmt: off + import logging import re import xml.etree.ElementTree as ET # nosec BXXX @@ -6,12 +9,13 @@ import requests # type: ignore -from search.config import SearchDocumentConfig -from search.construction_legislation import ( # noqa: E501 - construction_legislation_dataframe, -) -from search.utils.date import convert_date_string_to_obj -from search.utils.documents import ( # noqa: E501 +from construction_legislation import ( # noqa: E501 + construction_legislation_dataframe +) # noqa: E501 + +from fbr.search.config import SearchDocumentConfig +from fbr.search.utils.date import convert_date_string_to_obj +from fbr.search.utils.documents import ( # noqa: E501 generate_short_uuid, insert_or_update_document, ) diff --git a/fbr/search/public_gateway.py b/fbr/cache/public_gateway.py similarity index 96% rename from fbr/search/public_gateway.py rename to fbr/cache/public_gateway.py index 9de7452..d714876 100644 --- a/fbr/search/public_gateway.py +++ b/fbr/cache/public_gateway.py @@ -4,8 +4,8 @@ import requests # type: ignore -from search.utils.date import convert_date_string_to_obj -from search.utils.documents import ( # noqa: E501 +from fbr.search.utils.date import convert_date_string_to_obj +from fbr.search.utils.documents import ( # noqa: E501 generate_short_uuid, insert_or_update_document, ) diff --git a/fbr/cache/tasks.py b/fbr/cache/tasks.py new file mode 100644 index 0000000..2149cbc --- /dev/null +++ b/fbr/cache/tasks.py @@ -0,0 +1,18 @@ +from celery import shared_task + +from fbr.cache.legislation import Legislation +from fbr.cache.public_gateway import PublicGateway +from fbr.search.config import SearchDocumentConfig +from fbr.search.utils.documents import clear_all_documents + + +@shared_task() +def rebuild_cache(): + try: + clear_all_documents() + config = SearchDocumentConfig(search_query="", timeout=20) + Legislation().build_cache(config) + PublicGateway().build_cache(config) + return {"message": "rebuilt cache"} + except Exception as e: + return {"message": f"error clearing documents: {e}"} diff --git a/fbr/config/celery.py b/fbr/config/celery.py new file mode 100644 index 0000000..fcb920a --- /dev/null +++ b/fbr/config/celery.py @@ -0,0 +1,9 @@ +from celery import Celery + +app = Celery("fbr_celery") + +# Load settings from Django or directly +app.config_from_object("django.conf:settings", namespace="CELERY") + +# Auto-discover tasks in installed apps +app.autodiscover_tasks() diff --git a/fbr/config/settings/base.py b/fbr/config/settings/base.py index b2fece6..18fafe0 100644 --- a/fbr/config/settings/base.py +++ b/fbr/config/settings/base.py @@ -62,6 +62,7 @@ THIRD_PARTY_APPS: list = [ "webpack_loader", + "django_celery_beat", ] INSTALLED_APPS = DJANGO_APPS + LOCAL_APPS + THIRD_PARTY_APPS @@ -262,3 +263,9 @@ GOOGLE_ANALYTICS_TAG_MANAGER_ID = env( "GOOGLE_ANALYTICS_TAG_MANAGER_ID", default=None ) + +# Celery +CELERY_BROKER_URL = "redis://:6379/0" # TODO: actual value +CELERY_ACCEPT_CONTENT = ["json"] +CELERY_TASK_SERIALIZER = "json" +CELERY_RESULT_BACKEND = "redis://:6379/0" diff --git a/fbr/config/urls.py b/fbr/config/urls.py index d7a0553..8cb9817 100644 --- a/fbr/config/urls.py +++ b/fbr/config/urls.py @@ -11,13 +11,13 @@ from django.contrib import admin from django.urls import include, path -import core.views as core_views -import search.views as search_views +import fbr.core.views as core_views +import fbr.search.views as search_views -from search.config import SearchDocumentConfig -from search.models import DataResponseModel -from search.utils.documents import clear_all_documents -from search.utils.search import get_publisher_names, search +from fbr.search.config import SearchDocumentConfig +from fbr.search.models import DataResponseModel +from fbr.search.utils.documents import clear_all_documents +from fbr.search.utils.search import get_publisher_names, search urls_logger = logging.getLogger(__name__) @@ -94,8 +94,8 @@ def search(self, request, *args, **kwargs): class RebuildCacheViewSet(viewsets.ViewSet): @action(detail=False, methods=["post"], url_path="rebuild") def rebuild_cache(self, request, *args, **kwargs): - from search.legislation import Legislation - from search.public_gateway import PublicGateway + from fbr.cache.legislation import Legislation + from fbr.cache.public_gateway import PublicGateway tx_begin = time.time() try: diff --git a/fbr/management/commands/setup_periodic_task.py b/fbr/management/commands/setup_periodic_task.py new file mode 100644 index 0000000..cedf7e3 --- /dev/null +++ b/fbr/management/commands/setup_periodic_task.py @@ -0,0 +1,27 @@ +from django_celery_beat.models import CrontabSchedule, PeriodicTask + +from django.core.management.base import BaseCommand + + +class Command(BaseCommand): + help = "Setup periodic task for rebuilding cache" + + def handle(self, *args, **kwargs): + # Create or get the crontab schedule + schedule, created = CrontabSchedule.objects.get_or_create( + minute="0", hour="1" + ) + # Create the periodic task + task, created = PeriodicTask.objects.get_or_create( + crontab=schedule, + name="Rebuild Cache Daily", + task="fbr.cache.tasks.rebuild_cache", + ) + if created: + self.stdout.write( + self.style.SUCCESS("Periodic task created successfully.") + ) + else: + self.stdout.write( + self.style.WARNING("Periodic task already exists.") + ) diff --git a/fbr/search/tests/test_search.py b/fbr/search/tests/test_search.py index fcc1161..fa9d8d7 100644 --- a/fbr/search/tests/test_search.py +++ b/fbr/search/tests/test_search.py @@ -2,18 +2,18 @@ from unittest.mock import MagicMock, call, patch -from search.utils.search import create_search_query +from fbr.search.utils.search import create_search_query class TestCreateSearchQuery(unittest.TestCase): - @patch("search.utils.search.SearchQuery", autospec=True) + @patch("fbr.search.utils.search.SearchQuery", autospec=True) def test_single_word_query(self, mock_search_query): result = create_search_query("test") mock_search_query.assert_called_with("test", search_type="plain") self.assertEqual(result, mock_search_query.return_value) - @patch("search.utils.search.SearchQuery", autospec=True) + @patch("fbr.search.utils.search.SearchQuery", autospec=True) def test_implicit_and_search_operator_query(self, mock_search_query): # Mock SearchQuery instances mock_query1 = MagicMock(name="MockQuery1") @@ -35,7 +35,7 @@ def test_implicit_and_search_operator_query(self, mock_search_query): # Assert the AND operation was applied mock_query1.__and__.assert_called_once_with(mock_query2) - @patch("search.utils.search.SearchQuery", autospec=True) + @patch("fbr.search.utils.search.SearchQuery", autospec=True) def test_multiple_implicit_and_search_operator_query( self, mock_search_query ): @@ -61,7 +61,7 @@ def test_multiple_implicit_and_search_operator_query( # Assert the AND operation was applied mock_query1.__and__.assert_called_with(mock_query3) - @patch("search.utils.search.SearchQuery", autospec=True) + @patch("fbr.search.utils.search.SearchQuery", autospec=True) def test_and_search_operator_query(self, mock_search_query): # Mock SearchQuery instances mock_query1 = MagicMock(name="MockQuery1") @@ -83,7 +83,7 @@ def test_and_search_operator_query(self, mock_search_query): # Assert the AND operation was applied mock_query1.__and__.assert_called_once_with(mock_query2) - @patch("search.utils.search.SearchQuery", autospec=True) + @patch("fbr.search.utils.search.SearchQuery", autospec=True) def test_multiple_and_search_operator_query(self, mock_search_query): # Mock SearchQuery instances mock_query1 = MagicMock(name="MockQuery1") @@ -107,7 +107,7 @@ def test_multiple_and_search_operator_query(self, mock_search_query): # Assert the AND operation was applied mock_query1.__and__.assert_called_with(mock_query3) - @patch("search.utils.search.SearchQuery", autospec=True) + @patch("fbr.search.utils.search.SearchQuery", autospec=True) def test_or_search_operator_query(self, mock_search_query): # Mock SearchQuery instances mock_query1 = MagicMock(name="MockQuery1") @@ -129,7 +129,7 @@ def test_or_search_operator_query(self, mock_search_query): # Assert the AND operation was applied mock_query1.__or__.assert_called_once_with(mock_query2) - @patch("search.utils.search.SearchQuery", autospec=True) + @patch("fbr.search.utils.search.SearchQuery", autospec=True) def test_multple_or_search_operator_query(self, mock_search_query): # Mock SearchQuery instances mock_query1 = MagicMock(name="MockQuery1") @@ -153,7 +153,7 @@ def test_multple_or_search_operator_query(self, mock_search_query): # Assert the AND operation was applied mock_query1.__or__.assert_called_with(mock_query3) - @patch("search.utils.search.SearchQuery", autospec=True) + @patch("fbr.search.utils.search.SearchQuery", autospec=True) def test_multiple_or_search_operator_query(self, mock_search_query): # Mock SearchQuery instances mock_query1 = MagicMock(name="MockQuery1") @@ -177,7 +177,7 @@ def test_multiple_or_search_operator_query(self, mock_search_query): # Assert the AND operation was applied mock_query1.__or__.assert_called_with(mock_query3) - @patch("search.utils.search.SearchQuery", autospec=True) + @patch("fbr.search.utils.search.SearchQuery", autospec=True) def test_phrase_search_query(self, mock_search_query): result = create_search_query('"test trial"') mock_search_query.assert_called_with( @@ -185,7 +185,7 @@ def test_phrase_search_query(self, mock_search_query): ) self.assertEqual(result, mock_search_query.return_value) - @patch("search.utils.search.SearchQuery", autospec=True) + @patch("fbr.search.utils.search.SearchQuery", autospec=True) def test_and_multiple_single_single_phrase_search_query( self, mock_search_query ): @@ -221,7 +221,7 @@ def test_and_multiple_single_single_phrase_search_query( # Assert the AND operation was applied mock_query1.__and__.assert_called_with(mock_query5) - @patch("search.utils.search.SearchQuery", autospec=True) + @patch("fbr.search.utils.search.SearchQuery", autospec=True) def test_single_or_and_search_operator_query(self, mock_search_query): # Mock SearchQuery instances mock_query1 = MagicMock(name="MockQuery1") diff --git a/fbr/search/views.py b/fbr/search/views.py index 7a14d0e..992929b 100644 --- a/fbr/search/views.py +++ b/fbr/search/views.py @@ -3,15 +3,16 @@ import pandas as pd +from models import DataResponseModel +from utils.search import search, search_database + from django.conf import settings from django.core.serializers import serialize from django.http import HttpRequest, HttpResponse from django.shortcuts import render from django.views.decorators.http import require_http_methods -from search.config import SearchDocumentConfig -from search.models import DataResponseModel -from search.utils.search import search, search_database +from config import SearchDocumentConfig logger = logging.getLogger(__name__) diff --git a/poetry.lock b/poetry.lock index 3cec290..6b4c606 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. [[package]] name = "amqp" @@ -114,6 +114,44 @@ d = ["aiohttp (>=3.7.4)", "aiohttp (>=3.7.4,!=3.9.0)"] jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] uvloop = ["uvloop (>=0.15.2)"] +[[package]] +name = "boto3" +version = "1.35.71" +description = "The AWS SDK for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "boto3-1.35.71-py3-none-any.whl", hash = "sha256:e2969a246bb3208122b3c349c49cc6604c6fc3fc2b2f65d99d3e8ccd745b0c16"}, + {file = "boto3-1.35.71.tar.gz", hash = "sha256:3ed7172b3d4fceb6218bb0ec3668c4d40c03690939c2fca4f22bb875d741a07f"}, +] + +[package.dependencies] +botocore = ">=1.35.71,<1.36.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.10.0,<0.11.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.35.71" +description = "Low-level, data-driven core of boto 3." +optional = false +python-versions = ">=3.8" +files = [ + {file = "botocore-1.35.71-py3-none-any.whl", hash = "sha256:fc46e7ab1df3cef66dfba1633f4da77c75e07365b36f03bd64a3793634be8fc1"}, + {file = "botocore-1.35.71.tar.gz", hash = "sha256:f9fa058e0393660c3fe53c1e044751beb64b586def0bd2212448a7c328b0cbba"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""} + +[package.extras] +crt = ["awscrt (==0.22.0)"] + [[package]] name = "brotli" version = "1.1.0" @@ -225,6 +263,7 @@ click-plugins = ">=1.1.1" click-repl = ">=0.2.0" kombu = ">=5.3.4,<6.0" python-dateutil = ">=2.8.2" +redis = {version = ">=4.5.2,<4.5.5 || >4.5.5,<6.0.0", optional = true, markers = "extra == \"redis\""} tzdata = ">=2022.7" vine = ">=5.1.0,<6.0" @@ -472,6 +511,17 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "config" +version = "0.5.1" +description = "A hierarchical, easy-to-use, powerful configuration module for Python" +optional = false +python-versions = "*" +files = [ + {file = "config-0.5.1-py2.py3-none-any.whl", hash = "sha256:79ffa009ff2663cc8ca29e56bcec031c044609d4bafaa4f884132a413101ce84"}, + {file = "config-0.5.1.zip", hash = "sha256:2dd4a03aa383d30711d5a3325a1858de225328d61950a85be5b74c100f63016d"}, +] + [[package]] name = "coverage" version = "7.6.8" @@ -1000,6 +1050,17 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + [[package]] name = "kombu" version = "5.4.2" @@ -1013,6 +1074,7 @@ files = [ [package.dependencies] amqp = ">=5.1.1,<6.0.0" +redis = {version = ">=4.5.2,<4.5.5 || >4.5.5,<5.0.2 || >5.0.2", optional = true, markers = "extra == \"redis\""} tzdata = {version = "*", markers = "python_version >= \"3.9\""} vine = "5.1.0" @@ -2031,6 +2093,21 @@ files = [ {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, ] +[[package]] +name = "redis" +version = "5.2.0" +description = "Python client for Redis database and key-value store" +optional = false +python-versions = ">=3.8" +files = [ + {file = "redis-5.2.0-py3-none-any.whl", hash = "sha256:ae174f2bb3b1bf2b09d54bf3e51fbc1469cf6c10aa03e21141f51969801a7897"}, + {file = "redis-5.2.0.tar.gz", hash = "sha256:0b1087665a771b1ff2e003aa5bdd354f15a70c9e25d5a7dbf9c722c16528a7b0"}, +] + +[package.extras] +hiredis = ["hiredis (>=3.0.0)"] +ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==23.2.1)", "requests (>=2.31.0)"] + [[package]] name = "requests" version = "2.32.3" @@ -2052,6 +2129,23 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "s3transfer" +version = "0.10.4" +description = "An Amazon S3 Transfer Manager" +optional = false +python-versions = ">=3.8" +files = [ + {file = "s3transfer-0.10.4-py3-none-any.whl", hash = "sha256:244a76a24355363a68164241438de1b72f8781664920260c48465896b712a41e"}, + {file = "s3transfer-0.10.4.tar.gz", hash = "sha256:29edc09801743c21eb5ecbc617a152df41d3c287f67b615f73e5f750583666a7"}, +] + +[package.dependencies] +botocore = ">=1.33.2,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"] + [[package]] name = "sentry-sdk" version = "2.19.0" @@ -2205,6 +2299,16 @@ h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] +[[package]] +name = "utils" +version = "1.0.2" +description = "A grab-bag of utility functions and objects" +optional = false +python-versions = ">=3.6" +files = [ + {file = "utils-1.0.2.tar.gz", hash = "sha256:f4d5157e27e9d434006b5b52a1ec951a34e53e7ecaa145d43a153ec452eb5d9e"}, +] + [[package]] name = "vine" version = "5.1.0" @@ -2360,4 +2464,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "87c7280c68006f3bbe45f52bc82c65f4856a241a08ce7df7b0d231d85b881c0c" +content-hash = "e3d89c65cce1bf9db51bba3ff9b205b7241ec756f38c311f4ed6557f30101e8a" diff --git a/pyproject.toml b/pyproject.toml index c8123da..39afa53 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,6 +4,7 @@ version = "0.1.0" description = "Find business regulations" authors = ["Greg Barnes "] readme = "README.md" +packages = [{ include = "fbr" }] [tool.poetry.dependencies] python-environ = "^0.4.54" @@ -24,6 +25,11 @@ pandas = "^2.2.3" djangorestframework = "^3.15" grpcio = "^1.67.1" psycopg-c = "3.2.3" +config = "^0.5.1" +utils = "^1.0.2" +celery = {extras = ["redis"], version = "^5.4.0"} +kombu = {extras = ["redis"], version = "^5.4.2"} +boto3 = "^1.35.71" [tool.poetry.group.dev.dependencies] pre-commit = "^3.7.1" From 423ef501abaf961ba84725bfaceec133b04e7930 Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Thu, 28 Nov 2024 18:03:26 +0000 Subject: [PATCH 2/6] chore:add django-celery-beat and related dependencies Introduced django-celery-beat version 2.7.0 to manage periodic tasks more effectively with a database-backed approach. Updated the poetry.lock file to include necessary package dependencies such as cron-descriptor, django-timezone-field, and python-crontab, which support the new functionality. This change enhances task scheduling capabilities within Django applications. --- poetry.lock | 67 +++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 6b4c606..cc937ed 100644 --- a/poetry.lock +++ b/poetry.lock @@ -596,6 +596,20 @@ files = [ [package.extras] toml = ["tomli"] +[[package]] +name = "cron-descriptor" +version = "1.4.5" +description = "A Python library that converts cron expressions into human readable strings." +optional = false +python-versions = "*" +files = [ + {file = "cron_descriptor-1.4.5-py3-none-any.whl", hash = "sha256:736b3ae9d1a99bc3dbfc5b55b5e6e7c12031e7ba5de716625772f8b02dcd6013"}, + {file = "cron_descriptor-1.4.5.tar.gz", hash = "sha256:f51ce4ffc1d1f2816939add8524f206c376a42c87a5fca3091ce26725b3b1bca"}, +] + +[package.extras] +dev = ["polib"] + [[package]] name = "dbt-copilot-python" version = "0.2.2" @@ -698,6 +712,25 @@ tzdata = {version = "*", markers = "sys_platform == \"win32\""} argon2 = ["argon2-cffi (>=19.1.0)"] bcrypt = ["bcrypt"] +[[package]] +name = "django-celery-beat" +version = "2.7.0" +description = "Database-backed Periodic Tasks." +optional = false +python-versions = ">=3.8" +files = [ + {file = "django_celery_beat-2.7.0-py3-none-any.whl", hash = "sha256:851c680d8fbf608ca5fecd5836622beea89fa017bc2b3f94a5b8c648c32d84b1"}, + {file = "django_celery_beat-2.7.0.tar.gz", hash = "sha256:8482034925e09b698c05ad61c36ed2a8dbc436724a3fe119215193a4ca6dc967"}, +] + +[package.dependencies] +celery = ">=5.2.3,<6.0" +cron-descriptor = ">=1.2.32" +Django = ">=2.2,<5.2" +django-timezone-field = ">=5.0" +python-crontab = ">=2.3.4" +tzdata = "*" + [[package]] name = "django-log-formatter-asim" version = "0.0.4" @@ -713,6 +746,20 @@ files = [ django = ">=3,<5" pre-commit = ">=3.5.0,<4.0.0" +[[package]] +name = "django-timezone-field" +version = "7.0" +description = "A Django app providing DB, form, and REST framework fields for zoneinfo and pytz timezone objects." +optional = false +python-versions = "<4.0,>=3.8" +files = [ + {file = "django_timezone_field-7.0-py3-none-any.whl", hash = "sha256:3232e7ecde66ba4464abb6f9e6b8cc739b914efb9b29dc2cf2eee451f7cc2acb"}, + {file = "django_timezone_field-7.0.tar.gz", hash = "sha256:aa6f4965838484317b7f08d22c0d91a53d64e7bbbd34264468ae83d4023898a7"}, +] + +[package.dependencies] +Django = ">=3.2,<6.0" + [[package]] name = "django-webpack-loader" version = "3.1.1" @@ -1996,6 +2043,24 @@ pytest = ">=6.2.5" [package.extras] dev = ["pre-commit", "pytest-asyncio", "tox"] +[[package]] +name = "python-crontab" +version = "3.2.0" +description = "Python Crontab API" +optional = false +python-versions = "*" +files = [ + {file = "python_crontab-3.2.0-py3-none-any.whl", hash = "sha256:82cb9b6a312d41ff66fd3caf3eed7115c28c195bfb50711bc2b4b9592feb9fe5"}, + {file = "python_crontab-3.2.0.tar.gz", hash = "sha256:40067d1dd39ade3460b2ad8557c7651514cd3851deffff61c5c60e1227c5c36b"}, +] + +[package.dependencies] +python-dateutil = "*" + +[package.extras] +cron-description = ["cron-descriptor"] +cron-schedule = ["croniter"] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -2464,4 +2529,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "e3d89c65cce1bf9db51bba3ff9b205b7241ec756f38c311f4ed6557f30101e8a" +content-hash = "f0bc53e4854a76d20f643fe315c1f4b84135a12a941bfddc272c4773c255b2bf" diff --git a/pyproject.toml b/pyproject.toml index 39afa53..8848c11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ utils = "^1.0.2" celery = {extras = ["redis"], version = "^5.4.0"} kombu = {extras = ["redis"], version = "^5.4.2"} boto3 = "^1.35.71" +django-celery-beat = "^2.7.0" [tool.poetry.group.dev.dependencies] pre-commit = "^3.7.1" From f2ce5a04acd2bb2ac760c3a997ccb4bb0f2fe941 Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Fri, 29 Nov 2024 16:13:07 +0000 Subject: [PATCH 3/6] chore:add cache management script and improve rebuild process Introduce `manage_cache.py` for handling cache rebuilds, including a new script entry in `pyproject.toml`. Refactor imports and add logging to enhance clarity and maintainability in the codebase. Also, import `django-environ` and update the Makefile for additional cache rebuild command. --- Makefile | 7 +++++++ fbr/cache/legislation.py | 9 +++------ fbr/cache/manage_cache.py | 27 +++++++++++++++++++++++++++ fbr/cache/tasks.py | 6 +++++- fbr/search/apps.py | 2 +- fbr/search/utils/documents.py | 5 ++++- fbr/setup.py | 20 ++++++++++++++++++++ poetry.lock | 18 +++++++++++++++++- pyproject.toml | 4 ++++ 9 files changed, 88 insertions(+), 10 deletions(-) create mode 100644 fbr/cache/manage_cache.py create mode 100644 fbr/setup.py diff --git a/Makefile b/Makefile index 51cd311..829395f 100644 --- a/Makefile +++ b/Makefile @@ -124,3 +124,10 @@ isort: # Run isort secrets-baseline: # Generate a new secrets baseline file poetry run detect-secrets scan > .secrets.baseline + +rebuild_cache_man: + export PYTHONPATH=./fbr && \ + export DJANGO_SETTINGS_MODULE='fbr.config.settings.local' && \ + export DATABASE_URL=postgres://postgres:postgres@localhost:5432/fbr && \ + poetry install && \ + poetry run rebuild-cache diff --git a/fbr/cache/legislation.py b/fbr/cache/legislation.py index bbcefc8..8e522e2 100644 --- a/fbr/cache/legislation.py +++ b/fbr/cache/legislation.py @@ -9,12 +9,9 @@ import requests # type: ignore -from construction_legislation import ( # noqa: E501 - construction_legislation_dataframe -) # noqa: E501 - -from fbr.search.config import SearchDocumentConfig -from fbr.search.utils.date import convert_date_string_to_obj +from fbr.cache.construction_legislation import construction_legislation_dataframe # noqa: E501 +from fbr.search.config import SearchDocumentConfig # noqa: E501 +from fbr.search.utils.date import convert_date_string_to_obj # noqa: E501 from fbr.search.utils.documents import ( # noqa: E501 generate_short_uuid, insert_or_update_document, diff --git a/fbr/cache/manage_cache.py b/fbr/cache/manage_cache.py new file mode 100644 index 0000000..e7325db --- /dev/null +++ b/fbr/cache/manage_cache.py @@ -0,0 +1,27 @@ +# flake8: noqa +import os +import time + +import django + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local") + +django.setup() + +from fbr.cache.legislation import Legislation +from fbr.cache.public_gateway import PublicGateway +from fbr.search.config import SearchDocumentConfig +from fbr.search.utils.documents import clear_all_documents + + +def rebuild_cache(): + try: + start = time.time() + clear_all_documents() + config = SearchDocumentConfig(search_query="", timeout=20) + Legislation().build_cache(config) + PublicGateway().build_cache(config) + end = time.time() + return {"message": "rebuilt cache", "duration": round(end - start, 2)} + except Exception as e: + return {"message": f"error clearing documents: {e}"} diff --git a/fbr/cache/tasks.py b/fbr/cache/tasks.py index 2149cbc..3893148 100644 --- a/fbr/cache/tasks.py +++ b/fbr/cache/tasks.py @@ -1,3 +1,5 @@ +import time + from celery import shared_task from fbr.cache.legislation import Legislation @@ -9,10 +11,12 @@ @shared_task() def rebuild_cache(): try: + start = time.time() clear_all_documents() config = SearchDocumentConfig(search_query="", timeout=20) Legislation().build_cache(config) PublicGateway().build_cache(config) - return {"message": "rebuilt cache"} + end = time.time() + return {"message": "rebuilt cache", "duration": round(end - start, 2)} except Exception as e: return {"message": f"error clearing documents: {e}"} diff --git a/fbr/search/apps.py b/fbr/search/apps.py index 608fabf..f063631 100644 --- a/fbr/search/apps.py +++ b/fbr/search/apps.py @@ -13,6 +13,6 @@ class SearchConfig(AppConfig): """ - name = "search" + name = "fbr.search" verbose_name = "Find business regulations application functionality" default_auto_field = "django.db.models.BigAutoField" diff --git a/fbr/search/utils/documents.py b/fbr/search/utils/documents.py index 28207b6..949aba5 100644 --- a/fbr/search/utils/documents.py +++ b/fbr/search/utils/documents.py @@ -1,4 +1,5 @@ import base64 +import logging import re import uuid @@ -6,7 +7,9 @@ from django.db.models import QuerySet -from search.models import DataResponseModel, logger +from fbr.search.models import DataResponseModel + +logger = logging.getLogger(__name__) def clear_all_documents(): diff --git a/fbr/setup.py b/fbr/setup.py new file mode 100644 index 0000000..8a93cba --- /dev/null +++ b/fbr/setup.py @@ -0,0 +1,20 @@ +from setuptools import find_packages, setup + +setup( + name="fbr", + version="0.1", + packages=find_packages(), + install_requires=[ + # Add your package dependencies here + "requests", + "pandas", + "django", + "dj_database_url", + ], + entry_points={ + "console_scripts": [ + # Define command-line scripts here if needed + # e.g., 'my-command = fbr.module:function', + ], + }, +) diff --git a/poetry.lock b/poetry.lock index cc937ed..48c55ff 100644 --- a/poetry.lock +++ b/poetry.lock @@ -731,6 +731,22 @@ django-timezone-field = ">=5.0" python-crontab = ">=2.3.4" tzdata = "*" +[[package]] +name = "django-environ" +version = "0.11.2" +description = "A package that allows you to utilize 12factor inspired environment variables to configure your Django application." +optional = false +python-versions = ">=3.6,<4" +files = [ + {file = "django-environ-0.11.2.tar.gz", hash = "sha256:f32a87aa0899894c27d4e1776fa6b477e8164ed7f6b3e410a62a6d72caaf64be"}, + {file = "django_environ-0.11.2-py2.py3-none-any.whl", hash = "sha256:0ff95ab4344bfeff693836aa978e6840abef2e2f1145adff7735892711590c05"}, +] + +[package.extras] +develop = ["coverage[toml] (>=5.0a4)", "furo (>=2021.8.17b43,<2021.9.dev0)", "pytest (>=4.6.11)", "sphinx (>=3.5.0)", "sphinx-notfound-page"] +docs = ["furo (>=2021.8.17b43,<2021.9.dev0)", "sphinx (>=3.5.0)", "sphinx-notfound-page"] +testing = ["coverage[toml] (>=5.0a4)", "pytest (>=4.6.11)"] + [[package]] name = "django-log-formatter-asim" version = "0.0.4" @@ -2529,4 +2545,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "f0bc53e4854a76d20f643fe315c1f4b84135a12a941bfddc272c4773c255b2bf" +content-hash = "e006eca22e990a307b8c91016df6384b1581fba1619a6284a6758cfbb4220557" diff --git a/pyproject.toml b/pyproject.toml index 8848c11..614eecc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ celery = {extras = ["redis"], version = "^5.4.0"} kombu = {extras = ["redis"], version = "^5.4.2"} boto3 = "^1.35.71" django-celery-beat = "^2.7.0" +django-environ = "^0.11.2" [tool.poetry.group.dev.dependencies] pre-commit = "^3.7.1" @@ -101,3 +102,6 @@ ignore_errors = true [tool.flake8] max-line-length = 79 max-complexity = 10 + +[tool.poetry.scripts] +rebuild-cache = "fbr.cache.manage_cache:rebuild_cache" From 1ebf0f06fa50f76ced213da1266bd27d46a6dc73 Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Fri, 29 Nov 2024 18:11:07 +0000 Subject: [PATCH 4/6] refactor:import paths for search module Updated the import paths in several files to make them more consistent and streamlined by removing unnecessary "fbr" prefix. This change aims to improve readability and maintainability of the codebase without affecting the functional behavior. The entry script was also cleaned up by removing commented-out migration checks. --- fbr/cache/legislation.py | 7 ++++--- fbr/config/urls.py | 16 ++++++++-------- fbr/search/apps.py | 2 +- fbr/search/tests/test_search.py | 24 ++++++++++++------------ fbr/search/utils/documents.py | 5 +---- fbr/search/views.py | 7 +++---- local_deployment/entry.sh | 3 --- 7 files changed, 29 insertions(+), 35 deletions(-) diff --git a/fbr/cache/legislation.py b/fbr/cache/legislation.py index 8e522e2..03f419d 100644 --- a/fbr/cache/legislation.py +++ b/fbr/cache/legislation.py @@ -1,5 +1,6 @@ # isort: skip_file # fmt: off +# flake8: noqa import logging import re @@ -9,9 +10,9 @@ import requests # type: ignore -from fbr.cache.construction_legislation import construction_legislation_dataframe # noqa: E501 -from fbr.search.config import SearchDocumentConfig # noqa: E501 -from fbr.search.utils.date import convert_date_string_to_obj # noqa: E501 +from fbr.cache.construction_legislation import construction_legislation_dataframe +from fbr.search.config import SearchDocumentConfig +from fbr.search.utils.date import convert_date_string_to_obj from fbr.search.utils.documents import ( # noqa: E501 generate_short_uuid, insert_or_update_document, diff --git a/fbr/config/urls.py b/fbr/config/urls.py index 8cb9817..d7a0553 100644 --- a/fbr/config/urls.py +++ b/fbr/config/urls.py @@ -11,13 +11,13 @@ from django.contrib import admin from django.urls import include, path -import fbr.core.views as core_views -import fbr.search.views as search_views +import core.views as core_views +import search.views as search_views -from fbr.search.config import SearchDocumentConfig -from fbr.search.models import DataResponseModel -from fbr.search.utils.documents import clear_all_documents -from fbr.search.utils.search import get_publisher_names, search +from search.config import SearchDocumentConfig +from search.models import DataResponseModel +from search.utils.documents import clear_all_documents +from search.utils.search import get_publisher_names, search urls_logger = logging.getLogger(__name__) @@ -94,8 +94,8 @@ def search(self, request, *args, **kwargs): class RebuildCacheViewSet(viewsets.ViewSet): @action(detail=False, methods=["post"], url_path="rebuild") def rebuild_cache(self, request, *args, **kwargs): - from fbr.cache.legislation import Legislation - from fbr.cache.public_gateway import PublicGateway + from search.legislation import Legislation + from search.public_gateway import PublicGateway tx_begin = time.time() try: diff --git a/fbr/search/apps.py b/fbr/search/apps.py index f063631..608fabf 100644 --- a/fbr/search/apps.py +++ b/fbr/search/apps.py @@ -13,6 +13,6 @@ class SearchConfig(AppConfig): """ - name = "fbr.search" + name = "search" verbose_name = "Find business regulations application functionality" default_auto_field = "django.db.models.BigAutoField" diff --git a/fbr/search/tests/test_search.py b/fbr/search/tests/test_search.py index fa9d8d7..fcc1161 100644 --- a/fbr/search/tests/test_search.py +++ b/fbr/search/tests/test_search.py @@ -2,18 +2,18 @@ from unittest.mock import MagicMock, call, patch -from fbr.search.utils.search import create_search_query +from search.utils.search import create_search_query class TestCreateSearchQuery(unittest.TestCase): - @patch("fbr.search.utils.search.SearchQuery", autospec=True) + @patch("search.utils.search.SearchQuery", autospec=True) def test_single_word_query(self, mock_search_query): result = create_search_query("test") mock_search_query.assert_called_with("test", search_type="plain") self.assertEqual(result, mock_search_query.return_value) - @patch("fbr.search.utils.search.SearchQuery", autospec=True) + @patch("search.utils.search.SearchQuery", autospec=True) def test_implicit_and_search_operator_query(self, mock_search_query): # Mock SearchQuery instances mock_query1 = MagicMock(name="MockQuery1") @@ -35,7 +35,7 @@ def test_implicit_and_search_operator_query(self, mock_search_query): # Assert the AND operation was applied mock_query1.__and__.assert_called_once_with(mock_query2) - @patch("fbr.search.utils.search.SearchQuery", autospec=True) + @patch("search.utils.search.SearchQuery", autospec=True) def test_multiple_implicit_and_search_operator_query( self, mock_search_query ): @@ -61,7 +61,7 @@ def test_multiple_implicit_and_search_operator_query( # Assert the AND operation was applied mock_query1.__and__.assert_called_with(mock_query3) - @patch("fbr.search.utils.search.SearchQuery", autospec=True) + @patch("search.utils.search.SearchQuery", autospec=True) def test_and_search_operator_query(self, mock_search_query): # Mock SearchQuery instances mock_query1 = MagicMock(name="MockQuery1") @@ -83,7 +83,7 @@ def test_and_search_operator_query(self, mock_search_query): # Assert the AND operation was applied mock_query1.__and__.assert_called_once_with(mock_query2) - @patch("fbr.search.utils.search.SearchQuery", autospec=True) + @patch("search.utils.search.SearchQuery", autospec=True) def test_multiple_and_search_operator_query(self, mock_search_query): # Mock SearchQuery instances mock_query1 = MagicMock(name="MockQuery1") @@ -107,7 +107,7 @@ def test_multiple_and_search_operator_query(self, mock_search_query): # Assert the AND operation was applied mock_query1.__and__.assert_called_with(mock_query3) - @patch("fbr.search.utils.search.SearchQuery", autospec=True) + @patch("search.utils.search.SearchQuery", autospec=True) def test_or_search_operator_query(self, mock_search_query): # Mock SearchQuery instances mock_query1 = MagicMock(name="MockQuery1") @@ -129,7 +129,7 @@ def test_or_search_operator_query(self, mock_search_query): # Assert the AND operation was applied mock_query1.__or__.assert_called_once_with(mock_query2) - @patch("fbr.search.utils.search.SearchQuery", autospec=True) + @patch("search.utils.search.SearchQuery", autospec=True) def test_multple_or_search_operator_query(self, mock_search_query): # Mock SearchQuery instances mock_query1 = MagicMock(name="MockQuery1") @@ -153,7 +153,7 @@ def test_multple_or_search_operator_query(self, mock_search_query): # Assert the AND operation was applied mock_query1.__or__.assert_called_with(mock_query3) - @patch("fbr.search.utils.search.SearchQuery", autospec=True) + @patch("search.utils.search.SearchQuery", autospec=True) def test_multiple_or_search_operator_query(self, mock_search_query): # Mock SearchQuery instances mock_query1 = MagicMock(name="MockQuery1") @@ -177,7 +177,7 @@ def test_multiple_or_search_operator_query(self, mock_search_query): # Assert the AND operation was applied mock_query1.__or__.assert_called_with(mock_query3) - @patch("fbr.search.utils.search.SearchQuery", autospec=True) + @patch("search.utils.search.SearchQuery", autospec=True) def test_phrase_search_query(self, mock_search_query): result = create_search_query('"test trial"') mock_search_query.assert_called_with( @@ -185,7 +185,7 @@ def test_phrase_search_query(self, mock_search_query): ) self.assertEqual(result, mock_search_query.return_value) - @patch("fbr.search.utils.search.SearchQuery", autospec=True) + @patch("search.utils.search.SearchQuery", autospec=True) def test_and_multiple_single_single_phrase_search_query( self, mock_search_query ): @@ -221,7 +221,7 @@ def test_and_multiple_single_single_phrase_search_query( # Assert the AND operation was applied mock_query1.__and__.assert_called_with(mock_query5) - @patch("fbr.search.utils.search.SearchQuery", autospec=True) + @patch("search.utils.search.SearchQuery", autospec=True) def test_single_or_and_search_operator_query(self, mock_search_query): # Mock SearchQuery instances mock_query1 = MagicMock(name="MockQuery1") diff --git a/fbr/search/utils/documents.py b/fbr/search/utils/documents.py index 949aba5..28207b6 100644 --- a/fbr/search/utils/documents.py +++ b/fbr/search/utils/documents.py @@ -1,5 +1,4 @@ import base64 -import logging import re import uuid @@ -7,9 +6,7 @@ from django.db.models import QuerySet -from fbr.search.models import DataResponseModel - -logger = logging.getLogger(__name__) +from search.models import DataResponseModel, logger def clear_all_documents(): diff --git a/fbr/search/views.py b/fbr/search/views.py index 992929b..7a14d0e 100644 --- a/fbr/search/views.py +++ b/fbr/search/views.py @@ -3,16 +3,15 @@ import pandas as pd -from models import DataResponseModel -from utils.search import search, search_database - from django.conf import settings from django.core.serializers import serialize from django.http import HttpRequest, HttpResponse from django.shortcuts import render from django.views.decorators.http import require_http_methods -from config import SearchDocumentConfig +from search.config import SearchDocumentConfig +from search.models import DataResponseModel +from search.utils.search import search, search_database logger = logging.getLogger(__name__) diff --git a/local_deployment/entry.sh b/local_deployment/entry.sh index 245d9e4..b644ad4 100755 --- a/local_deployment/entry.sh +++ b/local_deployment/entry.sh @@ -11,8 +11,5 @@ npm run build echo "Collecting Static Files" python fbr/manage.py collectstatic --noinput -# echo "Check missing migrations" -# python prompt_payments/manage.py makemigrations --check --dry-run - echo "Starting server" python fbr/manage.py runserver 0.0.0.0:8080 From 07ac8e180a6315788d27559a336135c05868fd5d Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Sun, 1 Dec 2024 19:59:26 +0000 Subject: [PATCH 5/6] chore:minor refactoring and removed unused endpoint --- fbr/cache/public_gateway.py | 2 +- fbr/config/urls.py | 36 ------------------------------------ 2 files changed, 1 insertion(+), 37 deletions(-) diff --git a/fbr/cache/public_gateway.py b/fbr/cache/public_gateway.py index d714876..fd1eb06 100644 --- a/fbr/cache/public_gateway.py +++ b/fbr/cache/public_gateway.py @@ -86,7 +86,7 @@ def build_cache(self, config): row["date_valid"] = convert_date_string_to_obj( row.get("date_valid") ) - row["id"] = (generate_short_uuid(),) + row["id"] = generate_short_uuid() row["publisher_id"] = ( None diff --git a/fbr/config/urls.py b/fbr/config/urls.py index d7a0553..0ab1934 100644 --- a/fbr/config/urls.py +++ b/fbr/config/urls.py @@ -1,7 +1,6 @@ """Find business regulations URL configuration.""" import logging -import time from rest_framework import routers, serializers, status, viewsets from rest_framework.decorators import action @@ -14,9 +13,7 @@ import core.views as core_views import search.views as search_views -from search.config import SearchDocumentConfig from search.models import DataResponseModel -from search.utils.documents import clear_all_documents from search.utils.search import get_publisher_names, search urls_logger = logging.getLogger(__name__) @@ -91,38 +88,6 @@ def search(self, request, *args, **kwargs): ) -class RebuildCacheViewSet(viewsets.ViewSet): - @action(detail=False, methods=["post"], url_path="rebuild") - def rebuild_cache(self, request, *args, **kwargs): - from search.legislation import Legislation - from search.public_gateway import PublicGateway - - tx_begin = time.time() - try: - clear_all_documents() - config = SearchDocumentConfig(search_query="", timeout=20) - Legislation().build_cache(config) - PublicGateway().build_cache(config) - except Exception as e: - return Response( - data={"message": f"[urls] error clearing documents: {e}"}, - status=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) - - tx_end = time.time() - urls_logger.info( - f"time taken to rebuild cache: " - f"{round(tx_end - tx_begin, 2)} seconds" - ) - return Response( - data={ - "message": "rebuilt cache", - "duration": round(tx_end - tx_begin, 2), - }, - status=status.HTTP_200_OK, - ) - - class PublishersViewSet(viewsets.ViewSet): @action(detail=False, methods=["get"], url_path="publishers") def publishers(self, request, *args, **kwargs): @@ -151,7 +116,6 @@ def publishers(self, request, *args, **kwargs): # Routers provide an easy way of automatically determining the URL conf. router = routers.DefaultRouter() router.register(r"v1", DataResponseViewSet, basename="search") -router.register(r"v1/cache", RebuildCacheViewSet, basename="rebuild") router.register(r"v1/retrieve", PublishersViewSet, basename="publishers") urlpatterns = [ From 42248f917b867b5cc96ca36b05f7e74d122eea56 Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Sun, 1 Dec 2024 20:29:06 +0000 Subject: [PATCH 6/6] chore:improve search ID handling and remove redundant serialization Enhanced the search functionality by switching from `filter` to `get` for retrieving documents by ID, adding exception handling for non-existent IDs, and logging the search action. Removed unnecessary JSON serialization in the views to streamline data processing. These changes ensure more robust error management and simplify the data response in the user interface. --- fbr/search/utils/search.py | 6 +++++- fbr/search/views.py | 3 +-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/fbr/search/utils/search.py b/fbr/search/utils/search.py index bec07e8..a01f47e 100644 --- a/fbr/search/utils/search.py +++ b/fbr/search/utils/search.py @@ -68,7 +68,11 @@ def search_database( # If an id is provided, return the document with that id if config.id: - return DataResponseModel.objects.filter(id=config.id) + logger.debug(f"searching for document with id: {config.id}") + try: + return DataResponseModel.objects.get(id=config.id) + except DataResponseModel.DoesNotExist: + return DataResponseModel.objects.none() # Sanatize the query string query_str = sanitize_input(config.search_query) diff --git a/fbr/search/views.py b/fbr/search/views.py index 7a14d0e..9a7280b 100644 --- a/fbr/search/views.py +++ b/fbr/search/views.py @@ -4,7 +4,6 @@ import pandas as pd from django.conf import settings -from django.core.serializers import serialize from django.http import HttpRequest, HttpResponse from django.shortcuts import render from django.views.decorators.http import require_http_methods @@ -36,7 +35,7 @@ def document(request: HttpRequest, id) -> HttpResponse: try: queryset = search_database(config) - context["result"] = serialize("json", queryset) + context["result"] = queryset except Exception as e: logger.error("error fetching details: %s", e) context["error"] = f"error fetching details: {e}"