diff --git a/src/country_workspace/admin/__init__.py b/src/country_workspace/admin/__init__.py index 8176789..8cf37eb 100644 --- a/src/country_workspace/admin/__init__.py +++ b/src/country_workspace/admin/__init__.py @@ -16,7 +16,7 @@ from .role import UserRoleAdmin # noqa from .sync import SyncLog # noqa from .user import UserAdmin # noqa -from .kobo import KoboAsset # noqa +from country_workspace.contrib.kobo.admin import KoboAsset # noqa site.register(ContentType, admin_class=ContentTypeAdmin) site.register(Permission, admin_class=PermissionAdmin) diff --git a/src/country_workspace/admin/job.py b/src/country_workspace/admin/job.py index 1a14e4f..b9021c3 100644 --- a/src/country_workspace/admin/job.py +++ b/src/country_workspace/admin/job.py @@ -5,7 +5,7 @@ from django.http import HttpRequest from django_celery_boost.admin import CeleryTaskModelAdmin -from ..models import AsyncJob, KoboSyncJob +from ..models import AsyncJob from .base import BaseModelAdmin from .filters import FailedFilter @@ -29,7 +29,3 @@ def get_readonly_fields(self, request: "HttpRequest", obj: "AsyncJob | None" = N if obj: return "program", "batch", "owner", "local_status", "type", "action", "sentry_id" return super().get_readonly_fields(request, obj) - -@admin.register(KoboSyncJob) -class KoboSyncJobAdmin(CeleryTaskModelAdmin, BaseModelAdmin): - pass diff --git a/src/country_workspace/config/fragments/constance.py b/src/country_workspace/config/fragments/constance.py index b102843..771f14d 100644 --- a/src/country_workspace/config/fragments/constance.py +++ b/src/country_workspace/config/fragments/constance.py @@ -1,4 +1,5 @@ from .app import AURORA_API_TOKEN, AURORA_API_URL, HOPE_API_TOKEN, HOPE_API_URL, NEW_USER_DEFAULT_GROUP +from .kobo import KOBO_TOKEN, KOBO_BASE_URL CONSTANCE_BACKEND = "constance.backends.database.DatabaseBackend" @@ -46,8 +47,8 @@ "AURORA_API_URL": (AURORA_API_URL, "Aurora API Server address", str), "HOPE_API_TOKEN": (HOPE_API_TOKEN, "HOPE API Access Token", "write_only_input"), "HOPE_API_URL": (HOPE_API_URL, "HOPE API Server address", str), - "KOBO_API_TOKEN": ("", "Kobo API Access Token", "write_only_input"), - "KOBO_API_URL": ("", "Kobo API Server address", str), + "KOBO_TOKEN": (KOBO_TOKEN, "Kobo API Access Token", "write_only_input"), + "KOBO_BASE_URL": (KOBO_BASE_URL, "Kobo Server address", str), "CACHE_TIMEOUT": (86400, "Cache Redis TTL", int), "CACHE_BY_VERSION": (False, "Invalidate Cache on CW version change", bool), } @@ -60,7 +61,7 @@ "AURORA_API_URL", "HOPE_API_TOKEN", "HOPE_API_URL", - "KOBO_API_TOKEN", - "KOBO_API_URL", + "KOBO_TOKEN", + "KOBO_BASE_URL", ), } diff --git a/src/country_workspace/admin/kobo.py b/src/country_workspace/contrib/kobo/admin.py similarity index 50% rename from src/country_workspace/admin/kobo.py rename to src/country_workspace/contrib/kobo/admin.py index db9fdca..134c450 100644 --- a/src/country_workspace/admin/kobo.py +++ b/src/country_workspace/contrib/kobo/admin.py @@ -2,30 +2,14 @@ from django.contrib import admin -from .base import BaseModelAdmin -from ..models import KoboAsset -from ..models.kobo import KoboQuestion, KoboSubmission - - -class ReadOnlyInlineAdmin(admin.TabularInline): - can_create = False - can_change = False - can_delete = False - extra = 0 - -class KoboQuestionAdmin(ReadOnlyInlineAdmin): - model = KoboQuestion - - -class KoboSubmissionAdmin(ReadOnlyInlineAdmin): - model = KoboSubmission +from country_workspace.admin.base import BaseModelAdmin +from country_workspace.models import KoboAsset @admin.register(KoboAsset) class KoboAssetAdmin(BaseModelAdmin): list_display = ("uid", "name") exclude = ("programs",) - inlines = (KoboQuestionAdmin, KoboSubmissionAdmin) def has_add_permission(self, *args: Any, **kwargs: Any) -> bool: return False diff --git a/src/country_workspace/contrib/kobo/raw/__init__.py b/src/country_workspace/contrib/kobo/api/__init__.py similarity index 100% rename from src/country_workspace/contrib/kobo/raw/__init__.py rename to src/country_workspace/contrib/kobo/api/__init__.py diff --git a/src/country_workspace/contrib/kobo/auth.py b/src/country_workspace/contrib/kobo/api/auth.py similarity index 84% rename from src/country_workspace/contrib/kobo/auth.py rename to src/country_workspace/contrib/kobo/api/auth.py index aec0e44..7dd7df4 100644 --- a/src/country_workspace/contrib/kobo/auth.py +++ b/src/country_workspace/contrib/kobo/api/auth.py @@ -2,8 +2,8 @@ from requests.models import PreparedRequest -TOKEN = "Token" -AUTHORIZATION = 'Authorization' +TOKEN = "Token" # noqa: S105 +AUTHORIZATION = "Authorization" class Auth(AuthBase): def __init__(self, api_key: str) -> None: diff --git a/src/country_workspace/contrib/kobo/client.py b/src/country_workspace/contrib/kobo/api/client.py similarity index 84% rename from src/country_workspace/contrib/kobo/client.py rename to src/country_workspace/contrib/kobo/api/client.py index abcb32f..8f2e174 100644 --- a/src/country_workspace/contrib/kobo/client.py +++ b/src/country_workspace/contrib/kobo/api/client.py @@ -4,12 +4,12 @@ from black.linegen import partial from requests import Session, Response -from country_workspace.contrib.kobo.auth import Auth -from country_workspace.contrib.kobo.data import Submission, Asset, Question -from country_workspace.contrib.kobo.raw.common import ListResponse -from country_workspace.contrib.kobo.raw import asset as raw_asset, asset_list as raw_asset_list, common as raw_common -from country_workspace.contrib.kobo.raw import submission_list as raw_submission_list - +from country_workspace.contrib.kobo.api.auth import Auth +from country_workspace.contrib.kobo.api.data import Submission, Asset, Question +from country_workspace.contrib.kobo.api.raw.common import ListResponse +from country_workspace.contrib.kobo.api.raw import asset_list as raw_asset_list +from country_workspace.contrib.kobo.api.raw import asset as raw_asset, submission_list as raw_submission_list, \ + common as raw_common DataGetter = Callable[[str], Response] diff --git a/src/country_workspace/contrib/kobo/data.py b/src/country_workspace/contrib/kobo/api/data.py similarity index 89% rename from src/country_workspace/contrib/kobo/data.py rename to src/country_workspace/contrib/kobo/api/data.py index 9f9d553..29e4c03 100644 --- a/src/country_workspace/contrib/kobo/data.py +++ b/src/country_workspace/contrib/kobo/api/data.py @@ -7,8 +7,7 @@ from typing import Any from uuid import UUID -from country_workspace.contrib.kobo.raw import asset as raw_asset -from country_workspace.contrib.kobo.raw import submission_list as raw_submission_list +from country_workspace.contrib.kobo.api.raw import asset as raw_asset, submission_list as raw_submission_list class SurveyItemType(StrEnum): @@ -42,12 +41,11 @@ def extract_answer(self, in_: raw_submission_list.Submission, out: dict[str, Any roster[0][self.key] = in_.get(self.key) else: roster.append({self.key: in_.get(self.key)}) - else: - if roster_key in in_: - for i, item in enumerate(in_[roster_key]): - if len(roster) < i + 1: - roster.append({}) - roster[i][self.key] = item[self.key] + elif roster_key in in_: + for i, item in enumerate(in_[roster_key]): + if len(roster) < i + 1: + roster.append({}) + roster[i][self.key] = item[self.key] out[roster_key] = roster else: out[self.key] = in_.get(self.key) @@ -60,6 +58,9 @@ def key(self) -> str: def labels(self) -> list[str]: return self._raw["label"] + def __str__(self) -> str: + return f"Question: {' '.join(self.labels)}" + InAndOut = tuple[raw_submission_list.Submission, dict[str, Any]] @@ -132,3 +133,6 @@ def questions(self) -> list[Question]: @property def submissions(self) -> Generator[Submission, None, None]: yield from self._submissions(self.questions) + + def __str__(self) -> str: + return f"Asset: {self.name}" diff --git a/src/country_workspace/contrib/kobo/api/raw/__init__.py b/src/country_workspace/contrib/kobo/api/raw/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/country_workspace/contrib/kobo/raw/asset.py b/src/country_workspace/contrib/kobo/api/raw/asset.py similarity index 100% rename from src/country_workspace/contrib/kobo/raw/asset.py rename to src/country_workspace/contrib/kobo/api/raw/asset.py diff --git a/src/country_workspace/contrib/kobo/raw/asset_list.py b/src/country_workspace/contrib/kobo/api/raw/asset_list.py similarity index 69% rename from src/country_workspace/contrib/kobo/raw/asset_list.py rename to src/country_workspace/contrib/kobo/api/raw/asset_list.py index 063debe..dda5427 100644 --- a/src/country_workspace/contrib/kobo/raw/asset_list.py +++ b/src/country_workspace/contrib/kobo/api/raw/asset_list.py @@ -1,6 +1,6 @@ from typing import TypedDict -from country_workspace.contrib.kobo.raw.common import ListResponse +from country_workspace.contrib.kobo.api.raw.common import ListResponse class Asset(TypedDict): diff --git a/src/country_workspace/contrib/kobo/raw/common.py b/src/country_workspace/contrib/kobo/api/raw/common.py similarity index 100% rename from src/country_workspace/contrib/kobo/raw/common.py rename to src/country_workspace/contrib/kobo/api/raw/common.py diff --git a/src/country_workspace/contrib/kobo/raw/submission_list.py b/src/country_workspace/contrib/kobo/api/raw/submission_list.py similarity index 79% rename from src/country_workspace/contrib/kobo/raw/submission_list.py rename to src/country_workspace/contrib/kobo/api/raw/submission_list.py index 951cc0b..3063392 100644 --- a/src/country_workspace/contrib/kobo/raw/submission_list.py +++ b/src/country_workspace/contrib/kobo/api/raw/submission_list.py @@ -1,6 +1,6 @@ from typing import TypedDict -from country_workspace.contrib.kobo.raw.common import ListResponse +from country_workspace.contrib.kobo.api.raw.common import ListResponse class Attachment(TypedDict): diff --git a/src/country_workspace/contrib/kobo/forms.py b/src/country_workspace/contrib/kobo/forms.py new file mode 100644 index 0000000..c2ff29d --- /dev/null +++ b/src/country_workspace/contrib/kobo/forms.py @@ -0,0 +1,21 @@ +from django import forms + +from country_workspace.contrib.kobo.models import KoboAsset + + +class ImportKoboAssetsForm(forms.Form): + batch_name = forms.CharField(required=False, help_text="Label for this batch") + + +class ImportKoboDataForm(forms.Form): + batch_name = forms.CharField(required=False, help_text="Label for this batch") + individual_records_field = forms.CharField( + required=False, + initial="individual_questions", + help_text="Which field contains individual records", + ) + assets = forms.ModelMultipleChoiceField( + widget=forms.CheckboxSelectMultiple, + queryset=KoboAsset.objects.all(), + help_text="Which assets should be imported", + ) diff --git a/src/country_workspace/contrib/kobo/models.py b/src/country_workspace/contrib/kobo/models.py new file mode 100644 index 0000000..667b9f6 --- /dev/null +++ b/src/country_workspace/contrib/kobo/models.py @@ -0,0 +1,12 @@ +from django.db import models + +from country_workspace.models import Program + + +class KoboAsset(models.Model): + uid = models.CharField(primary_key=True, max_length=32, editable=False) + name = models.CharField(max_length=128, null=True, editable=False) + programs = models.ManyToManyField(Program) + + def __str__(self) -> str: + return self.name or "No name" diff --git a/src/country_workspace/contrib/kobo/sync.py b/src/country_workspace/contrib/kobo/sync.py new file mode 100644 index 0000000..c5a465f --- /dev/null +++ b/src/country_workspace/contrib/kobo/sync.py @@ -0,0 +1,48 @@ +from constance import config + +from country_workspace.contrib.kobo.api.client import Client as KoboClient +from country_workspace.models import AsyncJob, KoboAsset, Batch, Individual +from country_workspace.utils.fields import clean_field_name + + +def sync_assets(_: AsyncJob) -> dict[str, int]: + client = KoboClient(base_url=config.KOBO_BASE_URL, token=config.KOBO_TOKEN) + created_assets = 0 + updated_assets = 0 + for asset in client.assets: + asset_model, created = KoboAsset.objects.update_or_create(uid=asset.uid, defaults={"name": asset.name}) + if created: + created_assets += 1 + else: + updated_assets += 1 + + return {"created": created_assets, "updated": updated_assets} + + +def sync_data(job: AsyncJob) -> dict[str, int]: + batch = Batch.objects.create( + name=job.config["batch_name"], + program=job.program, + country_office=job.program.country_office, + imported_by=job.owner, + source=Batch.BatchSource.KOBO, + ) + individual_records_field = job.config["individual_records_field"] + client = KoboClient(base_url=config.KOBO_BASE_URL, token=config.KOBO_TOKEN) + assets = (asset for asset in client.assets if asset.uid in job.config["assets"]) + for asset in assets: + for submission in asset.submissions: + household_fields = {key: value for key, value in submission if key != individual_records_field} + household = batch.program.households.create(batch=batch, flex_fields={clean_field_name(key): value for key, value in household_fields.items()}) + individuals = [] + for individual in submission[individual_records_field]: + fullname = next((key for key in individual if key.startswith("given_name")), None) + individuals.append( + Individual( + batch=batch, + household_id=household.pk, + name=individual.get(fullname, ""), + flex_fields={clean_field_name(key): value for key, value in individual.items()}, + ), + ) + return {"households": 0, "individuals": 0} diff --git a/src/country_workspace/migrations/0002_alter_program_active_and_more.py b/src/country_workspace/migrations/0002_alter_program_active_and_more.py new file mode 100644 index 0000000..89d3929 --- /dev/null +++ b/src/country_workspace/migrations/0002_alter_program_active_and_more.py @@ -0,0 +1,65 @@ +# Generated by Django 5.1.3 on 2025-01-02 04:09 + +import django.db.models.deletion +import strategy_field.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("country_workspace", "0001_initial"), + ("hope_flex_fields", "0013_fielddefinition_validated_alter_datachecker_id_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="program", + name="active", + field=models.BooleanField( + default=False, help_text="Whether the program is active. Only active program are visible in the UI" + ), + ), + migrations.AlterField( + model_name="program", + name="beneficiary_validator", + field=strategy_field.fields.StrategyField( + blank=True, + default="country_workspace.validators.registry.NoopValidator", + help_text="Validator to use to validate the whole Household", + null=True, + ), + ), + migrations.AlterField( + model_name="program", + name="household_checker", + field=models.ForeignKey( + blank=True, + help_text="Checker to use with Household's records", + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="+", + to="hope_flex_fields.datachecker", + ), + ), + migrations.AlterField( + model_name="program", + name="individual_checker", + field=models.ForeignKey( + blank=True, + help_text="Checker to use with Individual's records", + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="+", + to="hope_flex_fields.datachecker", + ), + ), + migrations.CreateModel( + name="KoboAsset", + fields=[ + ("uid", models.CharField(editable=False, max_length=32, primary_key=True, serialize=False)), + ("name", models.CharField(editable=False, max_length=128, null=True)), + ("programs", models.ManyToManyField(to="country_workspace.program")), + ], + ), + ] diff --git a/src/country_workspace/migrations/0002_koboasset_kobosyncjob_kobosubmission.py b/src/country_workspace/migrations/0002_koboasset_kobosyncjob_kobosubmission.py deleted file mode 100644 index 49dbc10..0000000 --- a/src/country_workspace/migrations/0002_koboasset_kobosyncjob_kobosubmission.py +++ /dev/null @@ -1,95 +0,0 @@ -# Generated by Django 5.1.2 on 2024-11-21 09:02 - -import concurrency.fields -import django.db.models.deletion -from django.conf import settings -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("country_workspace", "0001_initial"), - ] - - operations = [ - migrations.CreateModel( - name="KoboAsset", - fields=[ - ("uid", models.CharField(max_length=32, primary_key=True, serialize=False)), - ], - ), - migrations.CreateModel( - name="KoboSyncJob", - fields=[ - ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), - ("version", concurrency.fields.AutoIncVersionField(default=0, help_text="record revision number")), - ( - "curr_async_result_id", - models.CharField( - blank=True, - editable=False, - help_text="Current (active) AsyncResult is", - max_length=36, - null=True, - ), - ), - ( - "last_async_result_id", - models.CharField( - blank=True, editable=False, help_text="Latest executed AsyncResult is", max_length=36, null=True - ), - ), - ("datetime_created", models.DateTimeField(auto_now_add=True, help_text="Creation date and time")), - ( - "datetime_queued", - models.DateTimeField( - blank=True, help_text="Queueing date and time", null=True, verbose_name="Queued At" - ), - ), - ( - "repeatable", - models.BooleanField( - blank=True, default=False, help_text="Indicate if the job can be repeated as-is" - ), - ), - ("celery_history", models.JSONField(blank=True, default=dict, editable=False)), - ("local_status", models.CharField(blank=True, default="", editable=False, max_length=100, null=True)), - ( - "group_key", - models.CharField( - blank=True, - editable=False, - help_text="Tasks with the same group key will not run in parallel", - max_length=255, - null=True, - ), - ), - ( - "owner", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.CASCADE, - related_name="%(app_label)s_%(class)s_jobs", - to=settings.AUTH_USER_MODEL, - ), - ), - ], - options={ - "abstract": False, - "default_permissions": ("add", "change", "delete", "view", "queue", "terminate", "inspect", "revoke"), - }, - ), - migrations.CreateModel( - name="KoboSubmission", - fields=[ - ("uuid", models.UUIDField(primary_key=True, serialize=False)), - ("data", models.JSONField()), - ( - "asset", - models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="country_workspace.koboasset"), - ), - ], - ), - ] diff --git a/src/country_workspace/migrations/0003_koboasset_programs.py b/src/country_workspace/migrations/0003_koboasset_programs.py deleted file mode 100644 index 54836fb..0000000 --- a/src/country_workspace/migrations/0003_koboasset_programs.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 5.1.2 on 2024-11-25 09:25 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("country_workspace", "0002_koboasset_kobosyncjob_kobosubmission"), - ] - - operations = [ - migrations.AddField( - model_name="koboasset", - name="programs", - field=models.ManyToManyField(to="country_workspace.program"), - ), - ] diff --git a/src/country_workspace/migrations/0004_koboasset_name.py b/src/country_workspace/migrations/0004_koboasset_name.py deleted file mode 100644 index cc3f604..0000000 --- a/src/country_workspace/migrations/0004_koboasset_name.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 5.1.2 on 2024-11-25 20:14 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("country_workspace", "0003_koboasset_programs"), - ] - - operations = [ - migrations.AddField( - model_name="koboasset", - name="name", - field=models.CharField(editable=False, max_length=128, null=True), - ), - ] diff --git a/src/country_workspace/migrations/0005_alter_koboasset_uid.py b/src/country_workspace/migrations/0005_alter_koboasset_uid.py deleted file mode 100644 index ef8844a..0000000 --- a/src/country_workspace/migrations/0005_alter_koboasset_uid.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 5.1.2 on 2024-11-25 20:19 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("country_workspace", "0004_koboasset_name"), - ] - - operations = [ - migrations.AlterField( - model_name="koboasset", - name="uid", - field=models.CharField(editable=False, max_length=32, primary_key=True, serialize=False), - ), - ] diff --git a/src/country_workspace/migrations/0006_alter_kobosubmission_data_koboquestion.py b/src/country_workspace/migrations/0006_alter_kobosubmission_data_koboquestion.py deleted file mode 100644 index 175ac4e..0000000 --- a/src/country_workspace/migrations/0006_alter_kobosubmission_data_koboquestion.py +++ /dev/null @@ -1,31 +0,0 @@ -# Generated by Django 5.1.2 on 2024-11-25 20:41 - -import django.db.models.deletion -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("country_workspace", "0005_alter_koboasset_uid"), - ] - - operations = [ - migrations.AlterField( - model_name="kobosubmission", - name="data", - field=models.JSONField(default=dict), - ), - migrations.CreateModel( - name="KoboQuestion", - fields=[ - ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), - ("key", models.CharField(editable=False, max_length=128, null=True)), - ("labels", models.JSONField(default=list)), - ( - "asset", - models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="country_workspace.koboasset"), - ), - ], - ), - ] diff --git a/src/country_workspace/migrations/0007_alter_koboquestion_unique_together.py b/src/country_workspace/migrations/0007_alter_koboquestion_unique_together.py deleted file mode 100644 index 241acc1..0000000 --- a/src/country_workspace/migrations/0007_alter_koboquestion_unique_together.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by Django 5.1.2 on 2024-11-25 21:54 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ("country_workspace", "0006_alter_kobosubmission_data_koboquestion"), - ] - - operations = [ - migrations.AlterUniqueTogether( - name="koboquestion", - unique_together={("asset", "key")}, - ), - ] diff --git a/src/country_workspace/models/__init__.py b/src/country_workspace/models/__init__.py index 2e01086..475deef 100644 --- a/src/country_workspace/models/__init__.py +++ b/src/country_workspace/models/__init__.py @@ -1,7 +1,7 @@ from .batch import Batch # noqa from .household import Household # noqa from .individual import Individual # noqa -from .jobs import AsyncJob, KoboSyncJob # noqa +from .jobs import AsyncJob # noqa from .locations import Area, AreaType, Country # noqa from .office import Office # noqa from .program import Program # noqa @@ -9,4 +9,4 @@ from .role import UserRole # noqa from .sync import SyncLog # noqa from .user import User # noqa -from .kobo import KoboAsset, KoboSubmission, KoboQuestion # noqa +from country_workspace.contrib.kobo.models import KoboAsset # noqa diff --git a/src/country_workspace/models/jobs.py b/src/country_workspace/models/jobs.py index e3c031b..03d7cf0 100644 --- a/src/country_workspace/models/jobs.py +++ b/src/country_workspace/models/jobs.py @@ -1,7 +1,6 @@ from typing import Any, Callable from django.apps import apps -from django.core.exceptions import ValidationError from django.db import models from django.utils.module_loading import import_string @@ -62,10 +61,3 @@ def execute(self) -> Any: if sid: self.sentry_id = sid self.save(update_fields=["sentry_id"]) - -class KoboSyncJob(CeleryTaskModel): - celery_task_name = "country_workspace.tasks.sync_kobo_assets_task" - - def clean(self) -> None: - if self.__class__.objects.exists() and not self.pk: - raise ValidationError(f"You can have only one {self.__class__.__name__} instance.") diff --git a/src/country_workspace/models/kobo.py b/src/country_workspace/models/kobo.py deleted file mode 100644 index fe291c5..0000000 --- a/src/country_workspace/models/kobo.py +++ /dev/null @@ -1,28 +0,0 @@ -from django.db import models - -from country_workspace.models import Program - - -class KoboAsset(models.Model): - uid = models.CharField(primary_key=True, max_length=32, editable=False) - name = models.CharField(max_length=128, null=True, editable=False) - programs = models.ManyToManyField(Program) - - def __str__(self) -> str: - return self.name or "No name" - - - -class KoboQuestion(models.Model): - asset = models.ForeignKey(KoboAsset, on_delete=models.CASCADE) - key = models.CharField(max_length=128, null=True, editable=False) - labels = models.JSONField(default=list) - - class Meta: - unique_together = ("asset", "key") - - -class KoboSubmission(models.Model): - uuid = models.UUIDField(primary_key=True) - asset = models.ForeignKey(KoboAsset, on_delete=models.CASCADE) - data = models.JSONField(default=dict) diff --git a/src/country_workspace/tasks.py b/src/country_workspace/tasks.py index 6c91cfe..abb918c 100644 --- a/src/country_workspace/tasks.py +++ b/src/country_workspace/tasks.py @@ -2,16 +2,12 @@ import logging from typing import TYPE_CHECKING, Any -from constance import config from django.core.cache import cache import sentry_sdk from redis_lock import Lock from country_workspace.config.celery import app -from country_workspace.contrib.kobo.client import Client as KoboClient -from country_workspace.models import AsyncJob, KoboAsset, KoboSubmission -from country_workspace.models.jobs import KoboSyncJob -from country_workspace.models.kobo import KoboQuestion +from country_workspace.models import AsyncJob logger = logging.getLogger(__name__) @@ -64,16 +60,3 @@ def sync_job_task(pk: int, version: int) -> dict[str, Any]: @app.task() def removed_expired_jobs(**kwargs: Any) -> None: AsyncJob.objects.filter(**kwargs).delete() - -@app.task -def sync_kobo_assets_task(job_id: int, version: int) -> None: - _ = KoboSyncJob.objects.get(pk=job_id, version=version) - client = KoboClient(base_url=config.KOBO_BASE_URL, token=config.KOBO_TOKEN) - for asset_data in client.assets: - asset_model, _ = KoboAsset.objects.update_or_create(uid=asset_data.uid, defaults={"name": asset_data.name}) - - for question_data in asset_data.questions: - KoboQuestion.objects.update_or_create(asset=asset_model, key=question_data.key, labels=question_data.labels) - - for submission_data in asset_data.submissions: - KoboSubmission.objects.update_or_create(uuid=submission_data.uuid, asset=asset_model, data=submission_data.data) diff --git a/src/country_workspace/workspaces/admin/program.py b/src/country_workspace/workspaces/admin/program.py index 2df8261..f562d77 100644 --- a/src/country_workspace/workspaces/admin/program.py +++ b/src/country_workspace/workspaces/admin/program.py @@ -19,6 +19,8 @@ from country_workspace.state import state from ...contrib.aurora.forms import ImportAuroraForm +from ...contrib.kobo.forms import ImportKoboDataForm, ImportKoboAssetsForm +from ...contrib.kobo.sync import sync_assets as sync_kobo_assets, sync_data as sync_kobo_data from ...datasources.rdi import import_from_rdi from ...models import AsyncJob from ...utils.flex_fields import get_checker_fields @@ -226,6 +228,8 @@ def import_data(self, request: HttpRequest, pk: str) -> "HttpResponse": context["media"] = Media(js=["admin/js/vendor/jquery/jquery.js", "workspace/js/import_data.js"], css={}) form_rdi = ImportFileForm(prefix="rdi") form_aurora = ImportAuroraForm(prefix="aurora") + form_kobo_assets = ImportKoboAssetsForm(prefix="kobo_assets") + form_kobo_data = ImportKoboDataForm(prefix="kobo_data") if request.method == "POST": match request.POST.get("_selected_tab"): @@ -235,11 +239,17 @@ def import_data(self, request: HttpRequest, pk: str) -> "HttpResponse": case "aurora": if not (form_aurora := self.import_aurora(request, program)): return HttpResponseRedirect(reverse("workspace:workspaces_countryasyncjob_changelist")) - case "kobo": - self.message_user(request, _("Not implemented")) + case "kobo-assets": + if not (form_kobo_data := self.import_kobo_assets(request, program)): + return HttpResponseRedirect(reverse("workspace:workspaces_countryasyncjob_changelist")) + case "kobo-data": + if not (form_kobo_data := self.import_kobo_data(request, program)): + return HttpResponseRedirect(reverse("workspace:workspaces_countryasyncjob_changelist")) context["form_rdi"] = form_rdi context["form_aurora"] = form_aurora + context["form_kobo_assets"] = form_kobo_assets + context["form_kobo_data"] = form_kobo_data return render(request, "workspace/program/import.html", context) @@ -287,3 +297,51 @@ def import_aurora(self, request: HttpRequest, program: "CountryProgram") -> "Imp ) return None return form + + def import_kobo_assets(self, request: HttpRequest, program: "CountryProgram") -> ImportKoboAssetsForm | None: + form = ImportKoboAssetsForm(request.POST, prefix="kobo_assets") + if form.is_valid(): + job: AsyncJob = AsyncJob.objects.create( + type=AsyncJob.JobType.TASK, + action=fqn(sync_kobo_assets), + file=None, + program=program, + owner=request.user, + config={ + "batch_name": form.cleaned_data["batch_name"] or BATCH_NAME_DEFAULT, + } + ) + job.queue() + self.message_user( + request, + _("The Kobo assets import task has been successfully queued. Job #{0}.").format(job.id), + level=messages.SUCCESS + ) + return None + + return form + + def import_kobo_data(self, request: HttpRequest, program: "CountryProgram") -> ImportKoboDataForm | None: + form = ImportKoboDataForm(request.POST, prefix="kobo_data") + if form.is_valid(): + job: AsyncJob = AsyncJob.objects.create( + type=AsyncJob.JobType.TASK, + action=fqn(sync_kobo_data), + file=None, + program=program, + owner=request.user, + config={ + "batch_name": form.cleaned_data["batch_name"] or BATCH_NAME_DEFAULT, + "assets": list(form.cleaned_data["assets"].values_list("uid", flat=True)), + "individual_records_field": form.cleaned_data["individual_records_field"], + } + ) + job.queue() + self.message_user( + request, + _("The Kobo data import task has been successfully queued. Job #{0}.").format(job.id), + level=messages.SUCCESS + ) + return None + + return form diff --git a/src/country_workspace/workspaces/templates/workspace/program/_import_kobo_assets.html b/src/country_workspace/workspaces/templates/workspace/program/_import_kobo_assets.html new file mode 100644 index 0000000..eb073d8 --- /dev/null +++ b/src/country_workspace/workspaces/templates/workspace/program/_import_kobo_assets.html @@ -0,0 +1,14 @@ +{% load i18n workspace_modify workspace_urls %} +
+ + {% csrf_token %} + + {{ form_kobo_assets.as_table }} +
+
+ + {% url opts|workspace_urlname:'change' original.pk as changelist_url %} + {% translate 'Close' %} +
+
diff --git a/src/country_workspace/workspaces/templates/workspace/program/_import_kobo_data.html b/src/country_workspace/workspaces/templates/workspace/program/_import_kobo_data.html new file mode 100644 index 0000000..d87b55b --- /dev/null +++ b/src/country_workspace/workspaces/templates/workspace/program/_import_kobo_data.html @@ -0,0 +1,14 @@ +{% load i18n workspace_modify workspace_urls %} +
+ + {% csrf_token %} + + {{ form_kobo_data.as_table }} +
+
+ + {% url opts|workspace_urlname:'change' original.pk as changelist_url %} + {% translate 'Close' %} +
+
diff --git a/src/country_workspace/workspaces/templates/workspace/program/import.html b/src/country_workspace/workspaces/templates/workspace/program/import.html index 0a3120e..66223d4 100644 --- a/src/country_workspace/workspaces/templates/workspace/program/import.html +++ b/src/country_workspace/workspaces/templates/workspace/program/import.html @@ -32,10 +32,18 @@ + @@ -48,8 +56,11 @@ - diff --git a/tests/contrib/kobo/test_auth.py b/tests/contrib/kobo/test_auth.py index 3060452..9ef1857 100644 --- a/tests/contrib/kobo/test_auth.py +++ b/tests/contrib/kobo/test_auth.py @@ -2,7 +2,7 @@ from requests.models import PreparedRequest -from country_workspace.contrib.kobo.auth import Auth, AUTHORIZATION, TOKEN +from country_workspace.contrib.kobo.api.auth import Auth, AUTHORIZATION, TOKEN def test_token_is_used() -> None: diff --git a/tests/contrib/kobo/test_kobo_client.py b/tests/contrib/kobo/test_kobo_client.py index a3da54e..62087ab 100644 --- a/tests/contrib/kobo/test_kobo_client.py +++ b/tests/contrib/kobo/test_kobo_client.py @@ -6,7 +6,7 @@ from requests.sessions import Session from requests.exceptions import Timeout -from country_workspace.contrib.kobo.client import _handle_paginated_response +from country_workspace.contrib.kobo.api.client import _handle_paginated_response SAMPLE_URL = "https://example.com"