From f2ce5a04acd2bb2ac760c3a997ccb4bb0f2fe941 Mon Sep 17 00:00:00 2001 From: Haresh Kainth Date: Fri, 29 Nov 2024 16:13:07 +0000 Subject: [PATCH] chore:add cache management script and improve rebuild process Introduce `manage_cache.py` for handling cache rebuilds, including a new script entry in `pyproject.toml`. Refactor imports and add logging to enhance clarity and maintainability in the codebase. Also, import `django-environ` and update the Makefile for additional cache rebuild command. --- Makefile | 7 +++++++ fbr/cache/legislation.py | 9 +++------ fbr/cache/manage_cache.py | 27 +++++++++++++++++++++++++++ fbr/cache/tasks.py | 6 +++++- fbr/search/apps.py | 2 +- fbr/search/utils/documents.py | 5 ++++- fbr/setup.py | 20 ++++++++++++++++++++ poetry.lock | 18 +++++++++++++++++- pyproject.toml | 4 ++++ 9 files changed, 88 insertions(+), 10 deletions(-) create mode 100644 fbr/cache/manage_cache.py create mode 100644 fbr/setup.py diff --git a/Makefile b/Makefile index 51cd311..829395f 100644 --- a/Makefile +++ b/Makefile @@ -124,3 +124,10 @@ isort: # Run isort secrets-baseline: # Generate a new secrets baseline file poetry run detect-secrets scan > .secrets.baseline + +rebuild_cache_man: + export PYTHONPATH=./fbr && \ + export DJANGO_SETTINGS_MODULE='fbr.config.settings.local' && \ + export DATABASE_URL=postgres://postgres:postgres@localhost:5432/fbr && \ + poetry install && \ + poetry run rebuild-cache diff --git a/fbr/cache/legislation.py b/fbr/cache/legislation.py index bbcefc8..8e522e2 100644 --- a/fbr/cache/legislation.py +++ b/fbr/cache/legislation.py @@ -9,12 +9,9 @@ import requests # type: ignore -from construction_legislation import ( # noqa: E501 - construction_legislation_dataframe -) # noqa: E501 - -from fbr.search.config import SearchDocumentConfig -from fbr.search.utils.date import convert_date_string_to_obj +from fbr.cache.construction_legislation import construction_legislation_dataframe # noqa: E501 +from fbr.search.config import SearchDocumentConfig # noqa: E501 +from fbr.search.utils.date import convert_date_string_to_obj # noqa: E501 from fbr.search.utils.documents import ( # noqa: E501 generate_short_uuid, insert_or_update_document, diff --git a/fbr/cache/manage_cache.py b/fbr/cache/manage_cache.py new file mode 100644 index 0000000..e7325db --- /dev/null +++ b/fbr/cache/manage_cache.py @@ -0,0 +1,27 @@ +# flake8: noqa +import os +import time + +import django + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local") + +django.setup() + +from fbr.cache.legislation import Legislation +from fbr.cache.public_gateway import PublicGateway +from fbr.search.config import SearchDocumentConfig +from fbr.search.utils.documents import clear_all_documents + + +def rebuild_cache(): + try: + start = time.time() + clear_all_documents() + config = SearchDocumentConfig(search_query="", timeout=20) + Legislation().build_cache(config) + PublicGateway().build_cache(config) + end = time.time() + return {"message": "rebuilt cache", "duration": round(end - start, 2)} + except Exception as e: + return {"message": f"error clearing documents: {e}"} diff --git a/fbr/cache/tasks.py b/fbr/cache/tasks.py index 2149cbc..3893148 100644 --- a/fbr/cache/tasks.py +++ b/fbr/cache/tasks.py @@ -1,3 +1,5 @@ +import time + from celery import shared_task from fbr.cache.legislation import Legislation @@ -9,10 +11,12 @@ @shared_task() def rebuild_cache(): try: + start = time.time() clear_all_documents() config = SearchDocumentConfig(search_query="", timeout=20) Legislation().build_cache(config) PublicGateway().build_cache(config) - return {"message": "rebuilt cache"} + end = time.time() + return {"message": "rebuilt cache", "duration": round(end - start, 2)} except Exception as e: return {"message": f"error clearing documents: {e}"} diff --git a/fbr/search/apps.py b/fbr/search/apps.py index 608fabf..f063631 100644 --- a/fbr/search/apps.py +++ b/fbr/search/apps.py @@ -13,6 +13,6 @@ class SearchConfig(AppConfig): """ - name = "search" + name = "fbr.search" verbose_name = "Find business regulations application functionality" default_auto_field = "django.db.models.BigAutoField" diff --git a/fbr/search/utils/documents.py b/fbr/search/utils/documents.py index 28207b6..949aba5 100644 --- a/fbr/search/utils/documents.py +++ b/fbr/search/utils/documents.py @@ -1,4 +1,5 @@ import base64 +import logging import re import uuid @@ -6,7 +7,9 @@ from django.db.models import QuerySet -from search.models import DataResponseModel, logger +from fbr.search.models import DataResponseModel + +logger = logging.getLogger(__name__) def clear_all_documents(): diff --git a/fbr/setup.py b/fbr/setup.py new file mode 100644 index 0000000..8a93cba --- /dev/null +++ b/fbr/setup.py @@ -0,0 +1,20 @@ +from setuptools import find_packages, setup + +setup( + name="fbr", + version="0.1", + packages=find_packages(), + install_requires=[ + # Add your package dependencies here + "requests", + "pandas", + "django", + "dj_database_url", + ], + entry_points={ + "console_scripts": [ + # Define command-line scripts here if needed + # e.g., 'my-command = fbr.module:function', + ], + }, +) diff --git a/poetry.lock b/poetry.lock index cc937ed..48c55ff 100644 --- a/poetry.lock +++ b/poetry.lock @@ -731,6 +731,22 @@ django-timezone-field = ">=5.0" python-crontab = ">=2.3.4" tzdata = "*" +[[package]] +name = "django-environ" +version = "0.11.2" +description = "A package that allows you to utilize 12factor inspired environment variables to configure your Django application." +optional = false +python-versions = ">=3.6,<4" +files = [ + {file = "django-environ-0.11.2.tar.gz", hash = "sha256:f32a87aa0899894c27d4e1776fa6b477e8164ed7f6b3e410a62a6d72caaf64be"}, + {file = "django_environ-0.11.2-py2.py3-none-any.whl", hash = "sha256:0ff95ab4344bfeff693836aa978e6840abef2e2f1145adff7735892711590c05"}, +] + +[package.extras] +develop = ["coverage[toml] (>=5.0a4)", "furo (>=2021.8.17b43,<2021.9.dev0)", "pytest (>=4.6.11)", "sphinx (>=3.5.0)", "sphinx-notfound-page"] +docs = ["furo (>=2021.8.17b43,<2021.9.dev0)", "sphinx (>=3.5.0)", "sphinx-notfound-page"] +testing = ["coverage[toml] (>=5.0a4)", "pytest (>=4.6.11)"] + [[package]] name = "django-log-formatter-asim" version = "0.0.4" @@ -2529,4 +2545,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "f0bc53e4854a76d20f643fe315c1f4b84135a12a941bfddc272c4773c255b2bf" +content-hash = "e006eca22e990a307b8c91016df6384b1581fba1619a6284a6758cfbb4220557" diff --git a/pyproject.toml b/pyproject.toml index 8848c11..614eecc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ celery = {extras = ["redis"], version = "^5.4.0"} kombu = {extras = ["redis"], version = "^5.4.2"} boto3 = "^1.35.71" django-celery-beat = "^2.7.0" +django-environ = "^0.11.2" [tool.poetry.group.dev.dependencies] pre-commit = "^3.7.1" @@ -101,3 +102,6 @@ ignore_errors = true [tool.flake8] max-line-length = 79 max-complexity = 10 + +[tool.poetry.scripts] +rebuild-cache = "fbr.cache.manage_cache:rebuild_cache"