cache management

pyproject.toml

@@ -16,7 +16,7 @@ dependencies = [
     "deprecation>=2.1.0",
     "dictdiffer>=0.9.0",
     "django-adminactions>=2.3.0",
-    "django-adminfilters>=2.5.1",
+    "django-adminfilters==2.5.1",
     "django-cacheops>=7.1",
     "django-celery-beat>=2.6.0",
     "django-celery-boost>=0.5.0",
@@ -36,6 +36,7 @@ dependencies = [
     "django-smart-env>=0.1.0",
     "django-storages[azure]>=1.14.4",
     "django-stubs-ext",
+    "django-sysinfo>=2.6.2",
     "django-tailwind>=3.8.0",
     "django>=5.1",
     "djangorestframework>=3.15.1",

src/country_workspace/admin/__init__.py

@@ -1,17 +1,28 @@
-from django.contrib.admin import site
+from django.contrib.admin.sites import site
+from django.contrib.auth.models import Permission
 from django.contrib.contenttypes.models import ContentType
 
-from smart_admin.smart_auth.admin import ContentTypeAdmin
+from smart_admin.console import panel_migrations, panel_redis, panel_sentry, panel_sysinfo
+from smart_admin.smart_auth.admin import ContentTypeAdmin, PermissionAdmin
 
 from .batch import BatchAdmin  # noqa
 from .household import HouseholdAdmin  # noqa
 from .individual import IndividualAdmin  # noqa
 from .job import AsyncJobAdmin  # noqa
 from .locations import AreaAdmin, AreaTypeAdmin, CountryAdmin  # noqa
 from .office import OfficeAdmin  # noqa
+from .panels.cache import panel_cache
 from .program import ProgramAdmin  # noqa
 from .role import UserRoleAdmin  # noqa
 from .sync import SyncLog  # noqa
 from .user import UserAdmin  # noqa
 
 site.register(ContentType, admin_class=ContentTypeAdmin)
+site.register(Permission, admin_class=PermissionAdmin)
+
+
+site.register_panel(panel_sentry)
+site.register_panel(panel_cache)
+site.register_panel(panel_sysinfo)
+site.register_panel(panel_migrations)
+site.register_panel(panel_redis)

src/country_workspace/admin/panels/cache.py

@@ -0,0 +1,40 @@
+from django import forms
+from django.shortcuts import render
+from django.utils.translation import gettext_lazy as _
+
+from country_workspace.cache.manager import cache_manager
+
+
+class CacheManagerForm(forms.Form):
+    pattern = forms.CharField()
+
+
+def panel_cache(self, request):
+    context = self.each_context(request)
+    client = cache_manager.get_redis_client()
+    limit_to = "*"
+
+    def _get_keys():
+        return list(client.scan_iter(f"*:cache:entry:{limit_to}"))
+
+    if request.method == "POST":
+        form = CacheManagerForm(request.POST)
+        if form.is_valid():
+            limit_to = form.cleaned_data["pattern"]
+            if "_delete" in request.POST:
+                to_delete = list(_get_keys())
+                if to_delete:
+                    client.delete(*to_delete)
+    else:
+        form = CacheManagerForm()
+
+    context["title"] = "Cache Manager"
+    context["form"] = form
+    cache_data = _get_keys()
+    context["cache_data"] = cache_data
+
+    return render(request, "smart_admin/panels/cache.html", context)
+
+
+panel_cache.verbose_name = _("Cache")  # type: ignore[attr-defined]
+panel_cache.url_name = "cache"  # type: ignore[attr-defined]

src/country_workspace/admin_site.py

@@ -0,0 +1,7 @@
+from smart_admin.site import SmartAdminSite
+
+
+class HCWAdminSite(SmartAdminSite):
+    site_header = "Workspace Admin"
+    site_title = "Workspace Admin Portal"
+    index_title = "Welcome to HOPE Workspace"

src/country_workspace/apps.py

@@ -1,7 +1,12 @@
 from django.apps import AppConfig
+from django.contrib.admin.apps import AdminConfig
 
 
-class Config(AppConfig):
+class HCWAdminConfig(AdminConfig):
+    default_site = "country_workspace.admin_site.HCWAdminSite"
+
+
+class HCWConfig(AppConfig):
     name = __name__.rpartition(".")[0]
     verbose_name = "Country Workspace"
 

src/country_workspace/cache/handlers.py

@@ -1,16 +1,21 @@
 from django.db.models.signals import post_save
 from django.dispatch import receiver
 
-from ..models import Household, Individual, Program
-from ..workspaces.models import CountryHousehold, CountryIndividual, CountryProgram
+from ..models import AsyncJob, Batch, Household, Individual, Program
+from ..workspaces.models import CountryAsyncJob, CountryBatch, CountryHousehold, CountryIndividual, CountryProgram
 from .manager import cache_manager
 
 
 @receiver(post_save)
 def update_cache(sender, instance, **kwargs):
+    program = None
     if isinstance(instance, (Household, Individual, CountryHousehold, CountryIndividual)):
         program = instance.program
-        cache_manager.incr_cache_version(program=program)
     elif isinstance(instance, (Program, CountryProgram)):
         program = instance
+    elif isinstance(instance, (AsyncJob, CountryAsyncJob)):
+        program = instance.program
+    elif isinstance(instance, (Batch, CountryBatch)):
+        program = instance.program
+    if program:
         cache_manager.incr_cache_version(program=program)

src/country_workspace/cache/manager.py

@@ -1,11 +1,13 @@
 import logging
 from typing import TYPE_CHECKING, Any, Optional
 
-from django.core.cache import cache
+from django.core.cache import caches
+from django.core.cache.backends.redis import RedisCacheClient
 from django.utils import timezone
 from django.utils.text import slugify
 
 from constance import config
+from redis_lock.django_cache import RedisCache
 from sentry_sdk import capture_exception
 
 from country_workspace import VERSION
@@ -19,12 +21,20 @@
 
 
 class CacheManager:
-    def __init__(self):
+    def __init__(self, prefix="cache"):
+        self.prefix = prefix
         self.active = True
         self.cw_version = "-"
         self.cache_timeout = 86400
         self.cache_by_version = False
 
+    def get_redis_client(self) -> RedisCacheClient:
+        return self.cache.client.get_client()
+
+    @property
+    def cache(self) -> RedisCache:
+        return caches["default"]
+
     def init(self):
         from . import handlers  # noqa
 
@@ -43,13 +53,13 @@ def invalidate(self, key):
     def retrieve(self, key):
         if not self.active:
             return None
-        data = cache.get(key)
+        data = self.cache.get(key)
         cache_get.send(CacheManager, key=key, hit=bool(data))
         return data
 
     def store(self, key: str, value: Any, timeout: int = 0, **kwargs):
         cache_set.send(self.__class__, key=key)
-        cache.set(key, value, timeout=timeout or self.cache_timeout, **kwargs)
+        self.cache.set(key, value, timeout=timeout or self.cache_timeout, **kwargs)
 
     def _get_version_key(self, office: "Optional[Office]" = None, program: "Optional[Program]" = None):
         if program:
@@ -58,31 +68,31 @@ def _get_version_key(self, office: "Optional[Office]" = None, program: "Optional
         elif office:
             program = None
 
-        parts = ["key", office.slug if office else "-", str(program.pk) if program else "-"]
+        parts = [self.prefix, "key", office.slug if office else "-", str(program.pk) if program else "-"]
         return ":".join(parts)
 
     def reset_cache_version(self, *, office: "Optional[Office]" = None, program: "Optional[Program]" = None):
         key = self._get_version_key(office, program)
-        cache.delete(key)
+        self.cache.delete(key)
 
     def get_cache_version(self, *, office: "Optional[Office]" = None, program: "Optional[Program]" = None):
         key = self._get_version_key(office, program)
-        return cache.get(key) or 1
+        return self.cache.get(key) or 1
 
     def incr_cache_version(self, *, office: "Optional[Office]" = None, program: "Optional[Program]" = None):
 
         key = self._get_version_key(office, program)
         try:
-            return cache.incr(key)
+            return self.cache.incr(key)
         except ValueError:
-            return cache.set(key, 2)
+            return self.cache.set(key, 2)
 
-    def build_key_from_request(self, request, prefix="view", *args):
+    def build_key(self, prefix, *parts):
         tenant = "t"
         version = "v"
         program = "p"
         ts = "ts"
-        if cache.get("cache_disabled"):
+        if self.cache.get("cache_disabled"):
             ts = str(timezone.now().toordinal())
 
         if state.tenant and state.program:
@@ -93,19 +103,14 @@ def build_key_from_request(self, request, prefix="view", *args):
             tenant = state.tenant.slug
             version = str(self.get_cache_version(office=state.tenant))
 
-        parts = [
-            prefix,
-            self.cw_version,
-            ts,
-            version,
-            tenant,
-            program,
-            slugify(request.path),
-            slugify(str(sorted(request.GET.items()))),
-            *[str(e) for e in args],
-        ]
+        parts = [self.prefix, "entry", prefix, self.cw_version, ts, version, tenant, program, *parts]
         return ":".join(parts)
 
+    def build_key_from_request(self, request, prefix="view", *args):
+        return self.build_key(
+            prefix, slugify(request.path), slugify(str(sorted(request.GET.items()))), *[str(e) for e in args]
+        )
+
     #
     # def store(self, request, value):
     #     key = self._build_key_from_request(request)

src/country_workspace/config/settings.py

@@ -25,9 +25,10 @@
     "django.contrib.sitemaps",
     "django.contrib.staticfiles",
     "django.contrib.postgres",
-    "django.contrib.admin",
+    "country_workspace.apps.HCWAdminConfig",
     # ddt
     "debug_toolbar",
+    "django_sysinfo",
     "flags",
     "reversion",
     "tailwind",
@@ -47,7 +48,7 @@
     "hope_smart_export",
     "smart_env",
     "country_workspace.security",
-    "country_workspace.apps.Config",
+    "country_workspace.apps.HCWConfig",
     "country_workspace.workspaces.apps.Config",
     "country_workspace.versioning",
     "country_workspace.cache",

src/country_workspace/config/urls.py

@@ -23,7 +23,7 @@
 
 urlpatterns += [path(r"", workspace.urls)]
 
-
-admin.site.site_header = "Workspace Admin"
-admin.site.site_title = "Workspace Admin Portal"
-admin.site.index_title = "Welcome to HOPE Workspace"
+#
+# admin.site.site_header = "Workspace Admin"
+# admin.site.site_title = "Workspace Admin Portal"
+# admin.site.index_title = "Welcome to HOPE Workspace"

src/country_workspace/datasources/rdi.py

@@ -43,9 +43,15 @@ def import_from_rdi(job: AsyncJob) -> dict[str, int]:
                             hh_ids[record[household_pk_col]] = hh.pk
                             ret["household"] += 1
                         elif sheet_index == 1:
+                            try:
+                                name = record[detail_column_label]
+                            except KeyError:
+                                raise Exception(
+                                    "Error in configuration. '%s' is not a valid column name" % detail_column_label
+                                )
                             job.program.individuals.create(
                                 batch=batch,
-                                name=raw_record[detail_column_label],
+                                name=name,
                                 household_id=hh_ids[record[household_pk_col]],
                                 flex_fields=record,
                             )

src/country_workspace/models/program.py

@@ -48,18 +48,34 @@ class Program(BaseModel):
     code = models.CharField(max_length=255, blank=True, null=True)
     status = models.CharField(max_length=10, choices=STATUS_CHOICE, db_index=True)
     sector = models.CharField(max_length=50, choices=SECTOR_CHOICE, db_index=True)
-    active = models.BooleanField(default=False)
+    active = models.BooleanField(
+        default=False, help_text=_("Whether the program is active. Only active program are visible in the UI")
+    )
 
     # Local Fields
     beneficiary_validator = StrategyField(
-        registry=beneficiary_validator_registry, default=fqn(NoopValidator), blank=True, null=True
+        registry=beneficiary_validator_registry,
+        default=fqn(NoopValidator),
+        blank=True,
+        null=True,
+        help_text="Validator to use to validate the whole Household",
     )
     household_checker = models.ForeignKey(
-        DataChecker, blank=True, null=True, on_delete=models.CASCADE, related_name="+"
+        DataChecker,
+        blank=True,
+        null=True,
+        on_delete=models.CASCADE,
+        related_name="+",
+        help_text="Checker to use with Household's records",
     )
 
     individual_checker = models.ForeignKey(
-        DataChecker, blank=True, null=True, on_delete=models.CASCADE, related_name="+"
+        DataChecker,
+        blank=True,
+        null=True,
+        on_delete=models.CASCADE,
+        related_name="+",
+        help_text="Checker to use with Individual's records",
     )
 
     household_search = models.TextField(default="name", help_text="Fields to use for searches")

src/country_workspace/versioning/management/manager.py

@@ -67,7 +67,7 @@ def _process_file(self, entry: Path):
         for func in funcs:
             try:
                 func()
-            except Exception as e:
+            except Exception as e:  # pragma: no cover
                 raise ScriptException(f"Error executing {entry.stem}.{func.__name__}") from e
         return funcs
 

src/country_workspace/web/static/smart_admin/console.scss

@@ -0,0 +1,17 @@
+// ADMIN
+#container {
+
+}
+.redis-cli{
+  input[type=text]{
+    width: 100%;
+  }
+  .code{
+    padding-left: 2px;
+    padding-right: 2px;
+    background-color: black;
+    color: white;
+    height: 500px;
+    overflow-scrolling: auto;
+  }
+}

src/country_workspace/web/static/smart_admin/smart_admin.css

@@ -0,0 +1 @@
+.panel{display:none}.panel.active{display:block}.console-buttons .button.selected{background-color:#114e6c}.sysinfo-results caption a{color:#fff}/*# sourceMappingURL=smart_admin.css.map */