Refactor code

src/country_workspace/admin/__init__.py

@@ -16,7 +16,7 @@
 from .role import UserRoleAdmin  # noqa
 from .sync import SyncLog  # noqa
 from .user import UserAdmin  # noqa
-from .kobo import KoboAsset  # noqa
+from country_workspace.contrib.kobo.admin import KoboAsset  # noqa
 
 site.register(ContentType, admin_class=ContentTypeAdmin)
 site.register(Permission, admin_class=PermissionAdmin)

src/country_workspace/admin/job.py

@@ -5,7 +5,7 @@
 from django.http import HttpRequest
 from django_celery_boost.admin import CeleryTaskModelAdmin
 
-from ..models import AsyncJob, KoboSyncJob
+from ..models import AsyncJob
 from .base import BaseModelAdmin
 from .filters import FailedFilter
 
@@ -29,7 +29,3 @@ def get_readonly_fields(self, request: "HttpRequest", obj: "AsyncJob | None" = N
         if obj:
             return "program", "batch", "owner", "local_status", "type", "action", "sentry_id"
         return super().get_readonly_fields(request, obj)
-
-@admin.register(KoboSyncJob)
-class KoboSyncJobAdmin(CeleryTaskModelAdmin, BaseModelAdmin):
-    pass

src/country_workspace/config/fragments/constance.py

@@ -1,4 +1,5 @@
 from .app import AURORA_API_TOKEN, AURORA_API_URL, HOPE_API_TOKEN, HOPE_API_URL, NEW_USER_DEFAULT_GROUP
+from .kobo import KOBO_TOKEN, KOBO_BASE_URL
 
 CONSTANCE_BACKEND = "constance.backends.database.DatabaseBackend"
 
@@ -46,8 +47,8 @@
     "AURORA_API_URL": (AURORA_API_URL, "Aurora API Server address", str),
     "HOPE_API_TOKEN": (HOPE_API_TOKEN, "HOPE API Access Token", "write_only_input"),
     "HOPE_API_URL": (HOPE_API_URL, "HOPE API Server address", str),
-    "KOBO_API_TOKEN": ("", "Kobo API Access Token", "write_only_input"),
-    "KOBO_API_URL": ("", "Kobo API Server address", str),
+    "KOBO_TOKEN": (KOBO_TOKEN, "Kobo API Access Token", "write_only_input"),
+    "KOBO_BASE_URL": (KOBO_BASE_URL, "Kobo Server address", str),
     "CACHE_TIMEOUT": (86400, "Cache Redis TTL", int),
     "CACHE_BY_VERSION": (False, "Invalidate Cache on CW version change", bool),
 }
@@ -60,7 +61,7 @@
         "AURORA_API_URL",
         "HOPE_API_TOKEN",
         "HOPE_API_URL",
-        "KOBO_API_TOKEN",
-        "KOBO_API_URL",
+        "KOBO_TOKEN",
+        "KOBO_BASE_URL",
     ),
 }

src/country_workspace/admin/kobo.py → src/country_workspace/contrib/kobo/admin.py

@@ -2,30 +2,14 @@
 
 from django.contrib import admin
 
-from .base import BaseModelAdmin
-from ..models import KoboAsset
-from ..models.kobo import KoboQuestion, KoboSubmission
-
-
-class ReadOnlyInlineAdmin(admin.TabularInline):
-    can_create = False
-    can_change = False
-    can_delete = False
-    extra = 0
-
-class KoboQuestionAdmin(ReadOnlyInlineAdmin):
-    model = KoboQuestion
-
-
-class KoboSubmissionAdmin(ReadOnlyInlineAdmin):
-    model = KoboSubmission
+from country_workspace.admin.base import BaseModelAdmin
+from country_workspace.models import KoboAsset
 
 
 @admin.register(KoboAsset)
 class KoboAssetAdmin(BaseModelAdmin):
     list_display = ("uid", "name")
     exclude = ("programs",)
-    inlines = (KoboQuestionAdmin, KoboSubmissionAdmin)
 
     def has_add_permission(self, *args: Any, **kwargs: Any) -> bool:
         return False

src/country_workspace/contrib/kobo/auth.py → src/country_workspace/contrib/kobo/api/auth.py

@@ -2,8 +2,8 @@
 from requests.models import PreparedRequest
 
 
-TOKEN = "Token"
-AUTHORIZATION = 'Authorization'
+TOKEN = "Token"  # noqa: S105
+AUTHORIZATION = "Authorization"
 
 class Auth(AuthBase):
     def __init__(self, api_key: str) -> None:

src/country_workspace/contrib/kobo/client.py → src/country_workspace/contrib/kobo/api/client.py

@@ -4,12 +4,12 @@
 from black.linegen import partial
 from requests import Session, Response
 
-from country_workspace.contrib.kobo.auth import Auth
-from country_workspace.contrib.kobo.data import Submission, Asset, Question
-from country_workspace.contrib.kobo.raw.common import ListResponse
-from country_workspace.contrib.kobo.raw import asset as raw_asset, asset_list as raw_asset_list, common as raw_common
-from country_workspace.contrib.kobo.raw import submission_list as raw_submission_list
-
+from country_workspace.contrib.kobo.api.auth import Auth
+from country_workspace.contrib.kobo.api.data import Submission, Asset, Question
+from country_workspace.contrib.kobo.api.raw.common import ListResponse
+from country_workspace.contrib.kobo.api.raw import asset_list as raw_asset_list
+from country_workspace.contrib.kobo.api.raw import asset as raw_asset, submission_list as raw_submission_list, \
+    common as raw_common
 
 DataGetter = Callable[[str], Response]
 

src/country_workspace/contrib/kobo/data.py → src/country_workspace/contrib/kobo/api/data.py

@@ -7,8 +7,7 @@
 from typing import Any
 from uuid import UUID
 
-from country_workspace.contrib.kobo.raw import asset as raw_asset
-from country_workspace.contrib.kobo.raw import submission_list as raw_submission_list
+from country_workspace.contrib.kobo.api.raw import asset as raw_asset, submission_list as raw_submission_list
 
 
 class SurveyItemType(StrEnum):
@@ -42,12 +41,11 @@ def extract_answer(self, in_: raw_submission_list.Submission, out: dict[str, Any
                     roster[0][self.key] = in_.get(self.key)
                 else:
                     roster.append({self.key: in_.get(self.key)})
-            else:
-                if roster_key in in_:
-                    for i, item in enumerate(in_[roster_key]):
-                        if len(roster) < i + 1:
-                            roster.append({})
-                        roster[i][self.key] = item[self.key]
+            elif roster_key in in_:
+                for i, item in enumerate(in_[roster_key]):
+                    if len(roster) < i + 1:
+                        roster.append({})
+                    roster[i][self.key] = item[self.key]
             out[roster_key] = roster
         else:
             out[self.key] = in_.get(self.key)
@@ -60,6 +58,9 @@ def key(self) -> str:
     def labels(self) -> list[str]:
         return self._raw["label"]
 
+    def __str__(self) -> str:
+        return f"Question: {' '.join(self.labels)}"
+
 
 InAndOut = tuple[raw_submission_list.Submission, dict[str, Any]]
 
@@ -132,3 +133,6 @@ def questions(self) -> list[Question]:
     @property
     def submissions(self) -> Generator[Submission, None, None]:
         yield from self._submissions(self.questions)
+
+    def __str__(self) -> str:
+        return f"Asset: {self.name}"

src/country_workspace/contrib/kobo/raw/asset_list.py → src/country_workspace/contrib/kobo/api/raw/asset_list.py

@@ -1,6 +1,6 @@
 from typing import TypedDict
 
-from country_workspace.contrib.kobo.raw.common import ListResponse
+from country_workspace.contrib.kobo.api.raw.common import ListResponse
 
 
 class Asset(TypedDict):

src/country_workspace/contrib/kobo/raw/submission_list.py → src/country_workspace/contrib/kobo/api/raw/submission_list.py

@@ -1,6 +1,6 @@
 from typing import TypedDict
 
-from country_workspace.contrib.kobo.raw.common import ListResponse
+from country_workspace.contrib.kobo.api.raw.common import ListResponse
 
 
 class Attachment(TypedDict):

src/country_workspace/contrib/kobo/forms.py

@@ -0,0 +1,21 @@
+from django import forms
+
+from country_workspace.contrib.kobo.models import KoboAsset
+
+
+class ImportKoboAssetsForm(forms.Form):
+    batch_name = forms.CharField(required=False, help_text="Label for this batch")
+
+
+class ImportKoboDataForm(forms.Form):
+    batch_name = forms.CharField(required=False, help_text="Label for this batch")
+    individual_records_field = forms.CharField(
+        required=False,
+        initial="individual_questions",
+        help_text="Which field contains individual records",
+    )
+    assets = forms.ModelMultipleChoiceField(
+        widget=forms.CheckboxSelectMultiple,
+        queryset=KoboAsset.objects.all(),
+        help_text="Which assets should be imported",
+    )

src/country_workspace/contrib/kobo/models.py

@@ -0,0 +1,12 @@
+from django.db import models
+
+from country_workspace.models import Program
+
+
+class KoboAsset(models.Model):
+    uid = models.CharField(primary_key=True, max_length=32, editable=False)
+    name = models.CharField(max_length=128, null=True, editable=False)
+    programs = models.ManyToManyField(Program)
+
+    def __str__(self) -> str:
+        return self.name or "No name"

src/country_workspace/contrib/kobo/sync.py

@@ -0,0 +1,48 @@
+from constance import config
+
+from country_workspace.contrib.kobo.api.client import Client as KoboClient
+from country_workspace.models import AsyncJob, KoboAsset, Batch, Individual
+from country_workspace.utils.fields import clean_field_name
+
+
+def sync_assets(_: AsyncJob) -> dict[str, int]:
+    client = KoboClient(base_url=config.KOBO_BASE_URL, token=config.KOBO_TOKEN)
+    created_assets = 0
+    updated_assets = 0
+    for asset in client.assets:
+        asset_model, created = KoboAsset.objects.update_or_create(uid=asset.uid, defaults={"name": asset.name})
+        if created:
+            created_assets += 1
+        else:
+            updated_assets += 1
+
+    return {"created": created_assets, "updated": updated_assets}
+
+
+def sync_data(job: AsyncJob) -> dict[str, int]:
+    batch = Batch.objects.create(
+        name=job.config["batch_name"],
+        program=job.program,
+        country_office=job.program.country_office,
+        imported_by=job.owner,
+        source=Batch.BatchSource.KOBO,
+    )
+    individual_records_field = job.config["individual_records_field"]
+    client = KoboClient(base_url=config.KOBO_BASE_URL, token=config.KOBO_TOKEN)
+    assets = (asset for asset in client.assets if asset.uid in job.config["assets"])
+    for asset in assets:
+        for submission in asset.submissions:
+            household_fields = {key: value for key, value in submission if key != individual_records_field}
+            household = batch.program.households.create(batch=batch, flex_fields={clean_field_name(key): value for key, value in household_fields.items()})
+            individuals = []
+            for individual in submission[individual_records_field]:
+                fullname = next((key for key in individual if key.startswith("given_name")), None)
+                individuals.append(
+                    Individual(
+                        batch=batch,
+                        household_id=household.pk,
+                        name=individual.get(fullname, ""),
+                        flex_fields={clean_field_name(key): value for key, value in individual.items()},
+                    ),
+                )
+    return {"households": 0, "individuals": 0}

src/country_workspace/migrations/0002_alter_program_active_and_more.py

@@ -0,0 +1,65 @@
+# Generated by Django 5.1.3 on 2025-01-02 04:09
+
+import django.db.models.deletion
+import strategy_field.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("country_workspace", "0001_initial"),
+        ("hope_flex_fields", "0013_fielddefinition_validated_alter_datachecker_id_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="program",
+            name="active",
+            field=models.BooleanField(
+                default=False, help_text="Whether the program is active. Only active program are visible in the UI"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="program",
+            name="beneficiary_validator",
+            field=strategy_field.fields.StrategyField(
+                blank=True,
+                default="country_workspace.validators.registry.NoopValidator",
+                help_text="Validator to use to validate the whole Household",
+                null=True,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="program",
+            name="household_checker",
+            field=models.ForeignKey(
+                blank=True,
+                help_text="Checker to use with Household's records",
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="+",
+                to="hope_flex_fields.datachecker",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="program",
+            name="individual_checker",
+            field=models.ForeignKey(
+                blank=True,
+                help_text="Checker to use with Individual's records",
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="+",
+                to="hope_flex_fields.datachecker",
+            ),
+        ),
+        migrations.CreateModel(
+            name="KoboAsset",
+            fields=[
+                ("uid", models.CharField(editable=False, max_length=32, primary_key=True, serialize=False)),
+                ("name", models.CharField(editable=False, max_length=128, null=True)),
+                ("programs", models.ManyToManyField(to="country_workspace.program")),
+            ],
+        ),
+    ]