MIGRATIONS RESET: updates permissions

src/country_workspace/admin/batch.py

@@ -3,19 +3,22 @@
 
 from admin_extra_buttons.buttons import LinkButton
 from admin_extra_buttons.decorators import link
+from adminfilters.autocomplete import AutoCompleteFilter, LinkedAutoCompleteFilter
 
 from ..models import Batch
 from .base import BaseModelAdmin
 
 
 @admin.register(Batch)
 class BatchAdmin(BaseModelAdmin):
-    list_display = ("name", "import_date", "imported_by", "program")
+    list_display = ("name", "import_date", "imported_by", "program", "source")
     list_filter = (
-        "country_office",
-        "program",
-        # ("country_office", LinkedAutoCompleteFilter.factory(parent=None)),
-        # ("program", LinkedAutoCompleteFilter.factory(parent="country_office")),
+        # "country_office",
+        # "program",
+        ("country_office", LinkedAutoCompleteFilter.factory(parent=None)),
+        ("program", LinkedAutoCompleteFilter.factory(parent="country_office")),
+        ("imported_by", AutoCompleteFilter),
+        "source",
     )
     readonly_fields = ("country_office", "program", "imported_by")
     search_fields = ("name",)

src/country_workspace/admin/filters.py

@@ -0,0 +1,74 @@
+from django.contrib.admin import SimpleListFilter
+from django.utils.translation import gettext as _
+
+
+class FailedFilter(SimpleListFilter):
+    title = "Status"
+    parameter_name = "failed"
+
+    def lookups(self, request, model_admin):
+        return (
+            ("f", _("Failed")),
+            ("s", _("Success")),
+        )
+
+    def get_title(self):
+        return self.title
+
+    def queryset(self, request, queryset):
+        if self.value() == "s":
+            return queryset.filter(sentry_id__isnull=True)
+        elif self.value() == "f":
+            return queryset.filter(sentry_id__isnull=False)
+        return queryset
+
+    def has_output(self):
+        return True
+
+    def html_attrs(self):
+        classes = f"adminfilters  {self.__class__.__name__.lower()}"
+        if self.value():
+            classes += " active"
+
+        return {
+            "class": classes,
+            "id": "_".join(self.expected_parameters()),
+        }
+
+
+class IsValidFilter(SimpleListFilter):
+    title = "Valid"
+    parameter_name = "valid"
+    # template = "workspace/adminfilters/combobox.html"
+
+    def lookups(self, request, model_admin):
+        return (
+            ("v", _("Valid")),
+            ("i", _("Invalid")),
+            ("u", _("Not Verified")),
+        )
+
+    def get_title(self):
+        return self.title
+
+    def queryset(self, request, queryset):
+        if self.value() == "v":
+            return queryset.filter(last_checked__isnull=False).filter(errors__iexact="{}")
+        elif self.value() == "i":
+            return queryset.filter(last_checked__isnull=False).exclude(errors__iexact="{}")
+        elif self.value() == "u":
+            return queryset.filter(last_checked__isnull=True)
+        return queryset
+
+    def has_output(self):
+        return True
+
+    def html_attrs(self):
+        classes = f"adminfilters  {self.__class__.__name__.lower()}"
+        if self.value():
+            classes += " active"
+
+        return {
+            "class": classes,
+            "id": "_".join(self.expected_parameters()),
+        }

src/country_workspace/admin/household.py

@@ -7,6 +7,7 @@
 
 from ..models import Household
 from .base import BaseModelAdmin
+from .filters import IsValidFilter
 
 
 @admin.register(Household)
@@ -16,6 +17,7 @@ class HouseholdAdmin(BaseModelAdmin):
         ("batch__country_office", LinkedAutoCompleteFilter.factory(parent=None)),
         ("batch__program", LinkedAutoCompleteFilter.factory(parent="batch__country_office")),
         ("batch", LinkedAutoCompleteFilter.factory(parent="batch__program")),
+        IsValidFilter,
     )
     # readonly_fields = ("country_office", "program")
     search_fields = ("name",)

src/country_workspace/admin/individual.py

@@ -7,6 +7,7 @@
 
 from ..models import Individual
 from .base import BaseModelAdmin
+from .filters import IsValidFilter
 
 
 @admin.register(Individual)
@@ -17,6 +18,8 @@ class IndividualAdmin(BaseModelAdmin):
     list_filter = (
         ("batch__country_office", LinkedAutoCompleteFilter.factory(parent=None)),
         ("batch__program", LinkedAutoCompleteFilter.factory(parent="batch__country_office")),
+        ("batch", LinkedAutoCompleteFilter.factory(parent="batch__program")),
+        IsValidFilter,
     )
     autocomplete_fields = ("batch",)
 

src/country_workspace/admin/job.py

@@ -1,59 +1,25 @@
 from typing import Optional, Sequence
 
 from django.contrib import admin
-from django.contrib.admin import SimpleListFilter
 from django.http import HttpRequest
-from django.utils.translation import gettext as _
 
 from adminfilters.autocomplete import AutoCompleteFilter, LinkedAutoCompleteFilter
 from django_celery_boost.admin import CeleryTaskModelAdmin
 
 from ..models import AsyncJob
 from .base import BaseModelAdmin
-
-
-class FailedFilter(SimpleListFilter):
-    title = "Status"
-    parameter_name = "failed"
-
-    def lookups(self, request, model_admin):
-        return (
-            ("f", _("Failed")),
-            ("s", _("Success")),
-        )
-
-    def get_title(self):
-        return self.title
-
-    def queryset(self, request, queryset):
-        if self.value() == "s":
-            return queryset.filter(sentry_id__isnull=True)
-        elif self.value() == "f":
-            return queryset.filter(sentry_id__isnull=False)
-        return queryset
-
-    def has_output(self):
-        return True
-
-    def html_attrs(self):
-        classes = f"adminfilters  {self.__class__.__name__.lower()}"
-        if self.value():
-            classes += " active"
-
-        return {
-            "class": classes,
-            "id": "_".join(self.expected_parameters()),
-        }
+from .filters import FailedFilter
 
 
 @admin.register(AsyncJob)
 class AsyncJobAdmin(CeleryTaskModelAdmin, BaseModelAdmin):
-    list_display = ("program", "type", "verbose_status")
+    list_display = ("program", "type", "verbose_status", "owner")
     autocomplete_fields = ("program", "owner")
     list_filter = (
         ("program__country_office", LinkedAutoCompleteFilter.factory(parent=None)),
         ("program", LinkedAutoCompleteFilter.factory(parent="program__country_office")),
         ("owner", AutoCompleteFilter),
+        "type",
         FailedFilter,
     )
 

src/country_workspace/contrib/aurora/sync.py

@@ -17,39 +17,29 @@ def sync_aurora_job(job: AsyncJob) -> dict[str, int]:
     Returns:
         dict[str, int]: A dictionary with counts of households and individuals created.
     """
+    batch_name = job.config["batch_name"]
+    batch = Batch.objects.create(
+        name=batch_name,
+        program=job.program,
+        country_office=job.program.country_office,
+        imported_by=job.owner,
+        source=Batch.BatchSource.RDI,
+    )
     total_hh = total_ind = 0
     client = AuroraClient()
     with atomic():
         for record in client.get("record"):
             for f_name, f_value in record["fields"].items():
                 if f_name == "household":
-                    hh = _create_household(job, f_value[0])
+                    hh = _create_household(batch, f_value[0])
                     total_hh += 1
                 elif f_name == "individuals":
-                    total_ind += len(_create_individuals(job, hh, f_value))
+                    total_ind += len(_create_individuals(hh, f_value, job.config.get("household_name_column", None)))
 
     return {"households": total_hh, "individuals": total_ind}
 
 
-def _create_batch(job: AsyncJob) -> Batch:
-    """
-    Creates a batch entity associated with the given job.
-
-    Args:
-        job (AsyncJob): The job instance containing the configuration for the batch creation.
-
-    Returns:
-        Batch: The newly created batch instance.
-    """
-    return Batch.objects.create(
-        name=job.config.get("batch_name"),
-        program=job.program,
-        country_office=job.program.country_office,
-        imported_by=job.owner,
-    )
-
-
-def _create_household(job: AsyncJob, fields: dict[str, Any]) -> Household:
+def _create_household(batch: Batch, fields: dict[str, Any]) -> Household:
     """
     Creates a household entity associated with the given job and batch.
 
@@ -60,15 +50,11 @@ def _create_household(job: AsyncJob, fields: dict[str, Any]) -> Household:
     Returns:
         Household: The newly created household instance.
     """
-    return job.program.households.create(
-        batch=job.batch, flex_fields={clean_field_name(k): v for k, v in fields.items()}
-    )
+    return batch.program.households.create(batch=batch, flex_fields={clean_field_name(k): v for k, v in fields.items()})
 
 
 def _create_individuals(
-    job: AsyncJob,
-    household: Household,
-    fields: list[dict[str, Any]],
+    household: Household, data: list[dict[str, Any]], household_name_column: str
 ) -> list[Individual]:
     """
     Creates individuals associated with a household and updates the household name if necessary.
@@ -82,24 +68,27 @@ def _create_individuals(
         list[Individual]: The list of newly created individual instances.
     """
     individuals = []
-    for individual in fields:
-
-        _update_household_name_from_individual(job, household, individual)
+    head_found = False
+    for individual in data:
+        if not head_found:
+            head_found = _update_household_name_from_individual(household, individual, household_name_column)
 
         fullname = next((k for k in individual if k.startswith("given_name")), None)
         individuals.append(
             Individual(
-                batch=job.batch,
+                batch=household.batch,
                 household_id=household.pk,
                 name=individual.get(fullname, ""),
                 flex_fields={clean_field_name(k): v for k, v in individual.items()},
             )
         )
 
-    return job.program.individuals.bulk_create(individuals)
+    return household.program.individuals.bulk_create(individuals)
 
 
-def _update_household_name_from_individual(job: AsyncJob, household: Household, individual: dict[str, Any]) -> None:
+def _update_household_name_from_individual(
+    household: Household, individual: dict[str, Any], household_name_column: str
+) -> bool:
     """
     Updates the household name based on an individual's relationship and name field.
 
@@ -116,6 +105,8 @@ def _update_household_name_from_individual(job: AsyncJob, household: Household,
     """
     if any(individual.get(k) == "head" for k in individual if k.startswith("relationship")):
         for k, v in individual.items():
-            if clean_field_name(k) == job.config["household_name_column"]:
-                job.program.households.filter(pk=household.pk).update(name=v)
-                break
+            if clean_field_name(k) == household_name_column:
+                household.name = v
+                household.save()
+                # household.program.households.filter(pk=household.pk).update(name=v)
+                return True

src/country_workspace/datasources/rdi.py

@@ -5,29 +5,30 @@
 
 from hope_smart_import.readers import open_xls_multi
 
-from country_workspace.models import AsyncJob, Batch, Household, Program
+from country_workspace.models import AsyncJob, Batch, Household
 from country_workspace.utils.fields import clean_field_name
 
 RDI = Union[str, io.BytesIO]
 
 
-def import_from_rdi_job(job: AsyncJob) -> dict[str, int]:
-    return import_from_rdi(
-        job.file,
-        program=job.program,
-        **job.config,
-    )
-
-
-def import_from_rdi(
-    rdi: RDI, batch: str, program: Program, household_pk_col: str, master_column_label: str, detail_column_label: str
-) -> dict[str, int]:
+def import_from_rdi(job: AsyncJob) -> dict[str, int]:
     ret = {"household": 0, "individual": 0}
     hh_ids = {}
     with atomic():
+        batch_name = job.config["batch_name"]
+        household_pk_col = job.config["household_pk_col"]
+        master_column_label = job.config["master_column_label"]
+        detail_column_label = job.config["detail_column_label"]
+        rdi = job.file
         # household_pk_col = form.cleaned_data["pk_column_name"]
         # total_hh = total_ind = 0
-        batch = Batch.objects.get(id=batch)
+        batch = Batch.objects.create(
+            name=batch_name,
+            program=job.program,
+            country_office=job.program.country_office,
+            imported_by=job.owner,
+            source=Batch.BatchSource.RDI,
+        )
         for sheet_index, sheet_generator in open_xls_multi(rdi, sheets=[0, 1]):
             for line, raw_record in enumerate(sheet_generator, 1):
                 record = {}
@@ -36,13 +37,13 @@ def import_from_rdi(
                 if record[household_pk_col]:
                     try:
                         if sheet_index == 0:
-                            hh: "Household" = program.households.create(
+                            hh: "Household" = job.program.households.create(
                                 batch=batch, name=raw_record[master_column_label], flex_fields=record
                             )
                             hh_ids[record[household_pk_col]] = hh.pk
                             ret["household"] += 1
                         elif sheet_index == 1:
-                            program.individuals.create(
+                            job.program.individuals.create(
                                 batch=batch,
                                 name=raw_record[detail_column_label],
                                 household_id=hh_ids[record[household_pk_col]],