diff --git a/src/country_workspace/admin/__init__.py b/src/country_workspace/admin/__init__.py index 16d872b..8176789 100644 --- a/src/country_workspace/admin/__init__.py +++ b/src/country_workspace/admin/__init__.py @@ -16,6 +16,7 @@ from .role import UserRoleAdmin # noqa from .sync import SyncLog # noqa from .user import UserAdmin # noqa +from .kobo import KoboAsset # noqa site.register(ContentType, admin_class=ContentTypeAdmin) site.register(Permission, admin_class=PermissionAdmin) diff --git a/src/country_workspace/admin/job.py b/src/country_workspace/admin/job.py index 0b95326..065d827 100644 --- a/src/country_workspace/admin/job.py +++ b/src/country_workspace/admin/job.py @@ -1,12 +1,11 @@ from typing import Optional, Sequence +from adminfilters.autocomplete import AutoCompleteFilter, LinkedAutoCompleteFilter from django.contrib import admin from django.http import HttpRequest - -from adminfilters.autocomplete import AutoCompleteFilter, LinkedAutoCompleteFilter from django_celery_boost.admin import CeleryTaskModelAdmin -from ..models import AsyncJob +from ..models import AsyncJob, KoboSyncJob from .base import BaseModelAdmin from .filters import FailedFilter @@ -25,5 +24,9 @@ class AsyncJobAdmin(CeleryTaskModelAdmin, BaseModelAdmin): def get_readonly_fields(self, request: "HttpRequest", obj: "Optional[AsyncJob]" = None) -> Sequence[str]: if obj: - return ("program", "batch", "owner", "local_status", "type", "action", "sentry_id") + return "program", "batch", "owner", "local_status", "type", "action", "sentry_id" return super().get_readonly_fields(request, obj) + +@admin.register(KoboSyncJob) +class KoboSyncJobAdmin(CeleryTaskModelAdmin, BaseModelAdmin): + pass diff --git a/src/country_workspace/admin/kobo.py b/src/country_workspace/admin/kobo.py new file mode 100644 index 0000000..db9fdca --- /dev/null +++ b/src/country_workspace/admin/kobo.py @@ -0,0 +1,37 @@ +from typing import Any + +from django.contrib import admin + +from .base import BaseModelAdmin +from ..models import KoboAsset +from ..models.kobo import KoboQuestion, KoboSubmission + + +class ReadOnlyInlineAdmin(admin.TabularInline): + can_create = False + can_change = False + can_delete = False + extra = 0 + +class KoboQuestionAdmin(ReadOnlyInlineAdmin): + model = KoboQuestion + + +class KoboSubmissionAdmin(ReadOnlyInlineAdmin): + model = KoboSubmission + + +@admin.register(KoboAsset) +class KoboAssetAdmin(BaseModelAdmin): + list_display = ("uid", "name") + exclude = ("programs",) + inlines = (KoboQuestionAdmin, KoboSubmissionAdmin) + + def has_add_permission(self, *args: Any, **kwargs: Any) -> bool: + return False + + def has_change_permission(self, *args: Any, **kwargs: Any) -> bool: + return False + + def has_delete_permission(self, *args: Any, **kwargs: Any) -> bool: + return False diff --git a/src/country_workspace/admin/program.py b/src/country_workspace/admin/program.py index d2009e4..9acbdb7 100644 --- a/src/country_workspace/admin/program.py +++ b/src/country_workspace/admin/program.py @@ -9,13 +9,18 @@ from ..cache.manager import cache_manager from ..compat.admin_extra_buttons import confirm_action -from ..models import Program +from ..models import Program, KoboAsset from .base import BaseModelAdmin if TYPE_CHECKING: from admin_extra_buttons.buttons import LinkButton +class KoboAssetInline(admin.TabularInline): + model = KoboAsset.programs.through + extra = 1 + + @admin.register(Program) class ProgramAdmin(BaseModelAdmin): list_display = ("name", "sector", "status", "active") @@ -23,12 +28,14 @@ class ProgramAdmin(BaseModelAdmin): list_filter = (("country_office", AutoCompleteFilter), "status", "active", "sector") ordering = ("name",) autocomplete_fields = ("country_office",) + inlines = (KoboAssetInline,) @button() def invalidate_cache(self, request: HttpRequest, pk: str) -> None: obj: [Program] = Program.objects.select_related("country_office").get(pk=pk) cache_manager.incr_cache_version(program=obj) + @link(change_list=False) def view_in_workspace(self, btn: "LinkButton") -> None: obj = btn.context["original"] diff --git a/src/country_workspace/config/__init__.py b/src/country_workspace/config/__init__.py index 2a4523c..3ccee92 100644 --- a/src/country_workspace/config/__init__.py +++ b/src/country_workspace/config/__init__.py @@ -214,6 +214,8 @@ class Group(Enum): "AZURE_CLIENT_KEY": (str, "", "", False, "Azure client key for SSO"), # "AZURE_CONNECTION_STRING": (str, ""), # "CV2DNN_PATH": (str, ""), + "KOBO_BASE_URL": (str, "", "", False, "Kobo API base URL"), + "KOBO_TOKEN": (str, "", "", False, "Kobo API token"), } env = SmartEnv(**CONFIG) diff --git a/src/country_workspace/config/fragments/kobo.py b/src/country_workspace/config/fragments/kobo.py new file mode 100644 index 0000000..8ecc14b --- /dev/null +++ b/src/country_workspace/config/fragments/kobo.py @@ -0,0 +1,4 @@ +from .. import env + +KOBO_BASE_URL = env("KOBO_BASE_URL") +KOBO_TOKEN = env("KOBO_TOKEN") diff --git a/src/country_workspace/config/settings.py b/src/country_workspace/config/settings.py index fb033e8..7d63ae5 100644 --- a/src/country_workspace/config/settings.py +++ b/src/country_workspace/config/settings.py @@ -242,6 +242,7 @@ from .fragments.rest_framework import * # noqa from .fragments.root import * # noqa from .fragments.sentry import * # noqa +from .fragments.kobo import * # noqa # from .fragments.smart_admin import * # noqa from .fragments.social_auth import * # noqa diff --git a/src/country_workspace/contrib/kobo/__init__.py b/src/country_workspace/contrib/kobo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/country_workspace/contrib/kobo/auth.py b/src/country_workspace/contrib/kobo/auth.py new file mode 100644 index 0000000..aec0e44 --- /dev/null +++ b/src/country_workspace/contrib/kobo/auth.py @@ -0,0 +1,14 @@ +from requests.auth import AuthBase +from requests.models import PreparedRequest + + +TOKEN = "Token" +AUTHORIZATION = 'Authorization' + +class Auth(AuthBase): + def __init__(self, api_key: str) -> None: + self._auth_header = f"{TOKEN} {api_key}" + + def __call__(self, request: PreparedRequest) -> PreparedRequest: + request.headers[AUTHORIZATION] = self._auth_header + return request diff --git a/src/country_workspace/contrib/kobo/client.py b/src/country_workspace/contrib/kobo/client.py new file mode 100644 index 0000000..abcb32f --- /dev/null +++ b/src/country_workspace/contrib/kobo/client.py @@ -0,0 +1,76 @@ +from collections.abc import Generator, Callable +from typing import cast + +from black.linegen import partial +from requests import Session, Response + +from country_workspace.contrib.kobo.auth import Auth +from country_workspace.contrib.kobo.data import Submission, Asset, Question +from country_workspace.contrib.kobo.raw.common import ListResponse +from country_workspace.contrib.kobo.raw import asset as raw_asset, asset_list as raw_asset_list, common as raw_common +from country_workspace.contrib.kobo.raw import submission_list as raw_submission_list + + +DataGetter = Callable[[str], Response] + + +def _handle_paginated_response[T, U](data_getter: DataGetter, + url: str, + collection_mapper: Callable[[ListResponse], list[T]], + item_mapper: Callable[[T], U]) -> Generator[U, None, None]: + while url: + response = data_getter(url) + response.raise_for_status() + data: ListResponse = response.json() + yield from map(item_mapper, collection_mapper(data)) + url = data["next"] + + +def _get_raw_asset_list(data: raw_common.ListResponse) -> list[raw_asset_list.Asset]: + return [ + datum for datum in + cast(raw_asset_list.AssetList, data)["results"] + if datum["asset_type"] == "survey" + ] + + +def _get_raw_submission_list(data: raw_common.ListResponse) -> list[raw_submission_list.Submission]: + return cast(raw_submission_list.SubmissionList, data)["results"] + + +def _get_asset_list(data_getter: DataGetter, url: str) -> Generator[Asset, None, None]: + return _handle_paginated_response(data_getter, + url, + _get_raw_asset_list, + partial(_get_asset, data_getter)) + +def _get_submission_list(data_getter: DataGetter, url: str, questions: list[Question]) -> Generator[Submission, None, None]: + return _handle_paginated_response( + data_getter, + url, + _get_raw_submission_list, + partial(Submission, data_getter, questions) + ) + + +def _get_asset(data_getter: DataGetter, raw: raw_asset_list.Asset) -> Asset: + response = data_getter(raw["url"]) + response.raise_for_status() + data: raw_asset.Asset = response.json() + return Asset(data, partial(_get_submission_list, data_getter, raw["data"])) + + +def _get_submission() -> Submission: + pass + + +class Client: + def __init__(self, *, base_url: str, token: str) -> None: + self.base_url = base_url + session = Session() + session.auth = Auth(token) + self.data_getter: DataGetter = session.get + + @property + def assets(self) -> Generator[Asset, None, None]: + yield from _get_asset_list(self.data_getter, f"{self.base_url}/api/v2/assets.json") diff --git a/src/country_workspace/contrib/kobo/data.py b/src/country_workspace/contrib/kobo/data.py new file mode 100644 index 0000000..9f9d553 --- /dev/null +++ b/src/country_workspace/contrib/kobo/data.py @@ -0,0 +1,134 @@ +from base64 import b64encode +from collections import UserDict +from collections.abc import Callable, Generator +from enum import StrEnum, auto +from functools import cached_property, reduce +from requests import Response +from typing import Any +from uuid import UUID + +from country_workspace.contrib.kobo.raw import asset as raw_asset +from country_workspace.contrib.kobo.raw import submission_list as raw_submission_list + + +class SurveyItemType(StrEnum): + START = auto() + END = auto() + BEGIN_GROUP = auto() + END_GROUP = auto() + BEGIN_REPEAT = auto() + END_REPEAT = auto() + + + +class Raw[T]: + def __init__(self, raw: T) -> None: + self._raw = raw + + +class Question(Raw[raw_asset.SurveyItem]): + def __init__(self, raw: raw_asset.SurveyItem, in_group: bool, in_roster: bool) -> None: + super().__init__(raw) + assert not (in_group and in_roster), "Cannot be both in group and roster" + self._in_group = in_group + self._in_roster = in_roster + + def extract_answer(self, in_: raw_submission_list.Submission, out: dict[str, Any]) -> None: + if self._in_roster: + roster_key, _ = self.key.split("/") + roster = out.get(roster_key, []) + if self.key in in_: + if roster: + roster[0][self.key] = in_.get(self.key) + else: + roster.append({self.key: in_.get(self.key)}) + else: + if roster_key in in_: + for i, item in enumerate(in_[roster_key]): + if len(roster) < i + 1: + roster.append({}) + roster[i][self.key] = item[self.key] + out[roster_key] = roster + else: + out[self.key] = in_.get(self.key) + + @property + def key(self) -> str: + return self._raw["$xpath"] + + @property + def labels(self) -> list[str]: + return self._raw["label"] + + +InAndOut = tuple[raw_submission_list.Submission, dict[str, Any]] + + +def _extract_answer(in_and_out: InAndOut, question: Question) -> InAndOut: + question.extract_answer(*in_and_out) + return in_and_out + + +def _download_attachments(data_getter: Callable[[str], Response], raw: raw_submission_list.Submission) -> None: + for attachment in raw["_attachments"]: + content = b64encode(data_getter(attachment["download_url"]).content).decode() + value = f"data:{attachment['mimetype']};base64,{content}" + key = attachment["question_xpath"] + if key in raw: + raw[key] = value + else: + parent, key = key.split("/") + parent, index = parent.split("[") + index = int(index.rstrip("]")) - 1 + raw[parent][index][f"{parent}/{key}"] = value + + +class Submission(Raw[raw_submission_list.Submission], UserDict): + def __init__(self, data_getter: Callable[[str], Response], questions: list[Question], raw: raw_submission_list.Submission) -> None: + Raw.__init__(self, raw) + _download_attachments(data_getter, self._raw) + _, answers = reduce(_extract_answer, questions, (raw, {})) + UserDict.__init__(self, answers) + + @property + def uuid(self) -> UUID: + return UUID(self._raw["_uuid"]) + + +class Asset(Raw[raw_asset.Asset]): + def __init__(self, raw: raw_asset.Asset, submissions: Callable[[list[Question]], Generator[Submission, None, None]]) -> None: + super().__init__(raw) + self._submissions = submissions + + @property + def uid(self) -> str: + return self._raw["uid"] + + @property + def name(self) -> str: + return self._raw["name"] + + @cached_property + def questions(self) -> list[Question]: + in_group = False + in_roster = False + questions = [] + for raw_question in self._raw["content"]["survey"]: + match raw_question["type"]: + case SurveyItemType.START | SurveyItemType.END: + pass + case SurveyItemType.BEGIN_GROUP: + in_group = True + case SurveyItemType.END_GROUP: + in_group = False + case SurveyItemType.BEGIN_REPEAT: + in_roster = True + case SurveyItemType.END_REPEAT: + in_roster = False + case _: + questions.append(Question(raw_question, in_group, in_roster)) + return questions + + @property + def submissions(self) -> Generator[Submission, None, None]: + yield from self._submissions(self.questions) diff --git a/src/country_workspace/contrib/kobo/raw/__init__.py b/src/country_workspace/contrib/kobo/raw/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/country_workspace/contrib/kobo/raw/asset.py b/src/country_workspace/contrib/kobo/raw/asset.py new file mode 100644 index 0000000..67729a9 --- /dev/null +++ b/src/country_workspace/contrib/kobo/raw/asset.py @@ -0,0 +1,14 @@ +from typing import TypedDict + +class SurveyItem(TypedDict("SurveyItem", {"$xpath": str})): + type: str + + +class Content(TypedDict): + survey: list[SurveyItem] + + +class Asset(TypedDict): + content: Content + name: str + uid: str diff --git a/src/country_workspace/contrib/kobo/raw/asset_list.py b/src/country_workspace/contrib/kobo/raw/asset_list.py new file mode 100644 index 0000000..063debe --- /dev/null +++ b/src/country_workspace/contrib/kobo/raw/asset_list.py @@ -0,0 +1,13 @@ +from typing import TypedDict + +from country_workspace.contrib.kobo.raw.common import ListResponse + + +class Asset(TypedDict): + data: str + url: str + asset_type: str + + +class AssetList(ListResponse): + results: list[Asset] diff --git a/src/country_workspace/contrib/kobo/raw/common.py b/src/country_workspace/contrib/kobo/raw/common.py new file mode 100644 index 0000000..17f54f6 --- /dev/null +++ b/src/country_workspace/contrib/kobo/raw/common.py @@ -0,0 +1,7 @@ +from typing import TypedDict + + +class ListResponse(TypedDict): + count: int + next: str | None + previous: str | None diff --git a/src/country_workspace/contrib/kobo/raw/submission_list.py b/src/country_workspace/contrib/kobo/raw/submission_list.py new file mode 100644 index 0000000..951cc0b --- /dev/null +++ b/src/country_workspace/contrib/kobo/raw/submission_list.py @@ -0,0 +1,18 @@ +from typing import TypedDict + +from country_workspace.contrib.kobo.raw.common import ListResponse + + +class Attachment(TypedDict): + download_url: str + mimetype: str + question_xpath: str + + +class Submission(TypedDict): + _uuid: str + _attachments: list[Attachment] + + +class SubmissionList(ListResponse): + results: list[Submission] diff --git a/src/country_workspace/migrations/0002_koboasset_kobosyncjob_kobosubmission.py b/src/country_workspace/migrations/0002_koboasset_kobosyncjob_kobosubmission.py new file mode 100644 index 0000000..49dbc10 --- /dev/null +++ b/src/country_workspace/migrations/0002_koboasset_kobosyncjob_kobosubmission.py @@ -0,0 +1,95 @@ +# Generated by Django 5.1.2 on 2024-11-21 09:02 + +import concurrency.fields +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("country_workspace", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="KoboAsset", + fields=[ + ("uid", models.CharField(max_length=32, primary_key=True, serialize=False)), + ], + ), + migrations.CreateModel( + name="KoboSyncJob", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("version", concurrency.fields.AutoIncVersionField(default=0, help_text="record revision number")), + ( + "curr_async_result_id", + models.CharField( + blank=True, + editable=False, + help_text="Current (active) AsyncResult is", + max_length=36, + null=True, + ), + ), + ( + "last_async_result_id", + models.CharField( + blank=True, editable=False, help_text="Latest executed AsyncResult is", max_length=36, null=True + ), + ), + ("datetime_created", models.DateTimeField(auto_now_add=True, help_text="Creation date and time")), + ( + "datetime_queued", + models.DateTimeField( + blank=True, help_text="Queueing date and time", null=True, verbose_name="Queued At" + ), + ), + ( + "repeatable", + models.BooleanField( + blank=True, default=False, help_text="Indicate if the job can be repeated as-is" + ), + ), + ("celery_history", models.JSONField(blank=True, default=dict, editable=False)), + ("local_status", models.CharField(blank=True, default="", editable=False, max_length=100, null=True)), + ( + "group_key", + models.CharField( + blank=True, + editable=False, + help_text="Tasks with the same group key will not run in parallel", + max_length=255, + null=True, + ), + ), + ( + "owner", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="%(app_label)s_%(class)s_jobs", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "abstract": False, + "default_permissions": ("add", "change", "delete", "view", "queue", "terminate", "inspect", "revoke"), + }, + ), + migrations.CreateModel( + name="KoboSubmission", + fields=[ + ("uuid", models.UUIDField(primary_key=True, serialize=False)), + ("data", models.JSONField()), + ( + "asset", + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="country_workspace.koboasset"), + ), + ], + ), + ] diff --git a/src/country_workspace/migrations/0003_koboasset_programs.py b/src/country_workspace/migrations/0003_koboasset_programs.py new file mode 100644 index 0000000..54836fb --- /dev/null +++ b/src/country_workspace/migrations/0003_koboasset_programs.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.2 on 2024-11-25 09:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("country_workspace", "0002_koboasset_kobosyncjob_kobosubmission"), + ] + + operations = [ + migrations.AddField( + model_name="koboasset", + name="programs", + field=models.ManyToManyField(to="country_workspace.program"), + ), + ] diff --git a/src/country_workspace/migrations/0004_koboasset_name.py b/src/country_workspace/migrations/0004_koboasset_name.py new file mode 100644 index 0000000..cc3f604 --- /dev/null +++ b/src/country_workspace/migrations/0004_koboasset_name.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.2 on 2024-11-25 20:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("country_workspace", "0003_koboasset_programs"), + ] + + operations = [ + migrations.AddField( + model_name="koboasset", + name="name", + field=models.CharField(editable=False, max_length=128, null=True), + ), + ] diff --git a/src/country_workspace/migrations/0005_alter_koboasset_uid.py b/src/country_workspace/migrations/0005_alter_koboasset_uid.py new file mode 100644 index 0000000..ef8844a --- /dev/null +++ b/src/country_workspace/migrations/0005_alter_koboasset_uid.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.2 on 2024-11-25 20:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("country_workspace", "0004_koboasset_name"), + ] + + operations = [ + migrations.AlterField( + model_name="koboasset", + name="uid", + field=models.CharField(editable=False, max_length=32, primary_key=True, serialize=False), + ), + ] diff --git a/src/country_workspace/migrations/0006_alter_kobosubmission_data_koboquestion.py b/src/country_workspace/migrations/0006_alter_kobosubmission_data_koboquestion.py new file mode 100644 index 0000000..175ac4e --- /dev/null +++ b/src/country_workspace/migrations/0006_alter_kobosubmission_data_koboquestion.py @@ -0,0 +1,31 @@ +# Generated by Django 5.1.2 on 2024-11-25 20:41 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("country_workspace", "0005_alter_koboasset_uid"), + ] + + operations = [ + migrations.AlterField( + model_name="kobosubmission", + name="data", + field=models.JSONField(default=dict), + ), + migrations.CreateModel( + name="KoboQuestion", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("key", models.CharField(editable=False, max_length=128, null=True)), + ("labels", models.JSONField(default=list)), + ( + "asset", + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="country_workspace.koboasset"), + ), + ], + ), + ] diff --git a/src/country_workspace/migrations/0007_alter_koboquestion_unique_together.py b/src/country_workspace/migrations/0007_alter_koboquestion_unique_together.py new file mode 100644 index 0000000..241acc1 --- /dev/null +++ b/src/country_workspace/migrations/0007_alter_koboquestion_unique_together.py @@ -0,0 +1,17 @@ +# Generated by Django 5.1.2 on 2024-11-25 21:54 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("country_workspace", "0006_alter_kobosubmission_data_koboquestion"), + ] + + operations = [ + migrations.AlterUniqueTogether( + name="koboquestion", + unique_together={("asset", "key")}, + ), + ] diff --git a/src/country_workspace/models/__init__.py b/src/country_workspace/models/__init__.py index f0748e0..2e01086 100644 --- a/src/country_workspace/models/__init__.py +++ b/src/country_workspace/models/__init__.py @@ -1,7 +1,7 @@ from .batch import Batch # noqa from .household import Household # noqa from .individual import Individual # noqa -from .jobs import AsyncJob # noqa +from .jobs import AsyncJob, KoboSyncJob # noqa from .locations import Area, AreaType, Country # noqa from .office import Office # noqa from .program import Program # noqa @@ -9,3 +9,4 @@ from .role import UserRole # noqa from .sync import SyncLog # noqa from .user import User # noqa +from .kobo import KoboAsset, KoboSubmission, KoboQuestion # noqa diff --git a/src/country_workspace/models/jobs.py b/src/country_workspace/models/jobs.py index 0d3c224..b54574b 100644 --- a/src/country_workspace/models/jobs.py +++ b/src/country_workspace/models/jobs.py @@ -1,4 +1,5 @@ from django.apps import apps +from django.core.exceptions import ValidationError from django.db import models from django.utils.module_loading import import_string @@ -64,3 +65,10 @@ def execute(self): if sid: self.sentry_id = sid self.save(update_fields=["sentry_id"]) + +class KoboSyncJob(CeleryTaskModel): + celery_task_name = "country_workspace.tasks.sync_kobo_assets_task" + + def clean(self) -> None: + if self.__class__.objects.exists() and not self.pk: + raise ValidationError(f"You can have only one {self.__class__.__name__} instance.") diff --git a/src/country_workspace/models/kobo.py b/src/country_workspace/models/kobo.py new file mode 100644 index 0000000..fe291c5 --- /dev/null +++ b/src/country_workspace/models/kobo.py @@ -0,0 +1,28 @@ +from django.db import models + +from country_workspace.models import Program + + +class KoboAsset(models.Model): + uid = models.CharField(primary_key=True, max_length=32, editable=False) + name = models.CharField(max_length=128, null=True, editable=False) + programs = models.ManyToManyField(Program) + + def __str__(self) -> str: + return self.name or "No name" + + + +class KoboQuestion(models.Model): + asset = models.ForeignKey(KoboAsset, on_delete=models.CASCADE) + key = models.CharField(max_length=128, null=True, editable=False) + labels = models.JSONField(default=list) + + class Meta: + unique_together = ("asset", "key") + + +class KoboSubmission(models.Model): + uuid = models.UUIDField(primary_key=True) + asset = models.ForeignKey(KoboAsset, on_delete=models.CASCADE) + data = models.JSONField(default=dict) diff --git a/src/country_workspace/tasks.py b/src/country_workspace/tasks.py index 8dbb3a2..5f5ae5f 100644 --- a/src/country_workspace/tasks.py +++ b/src/country_workspace/tasks.py @@ -2,13 +2,16 @@ import logging from typing import TYPE_CHECKING, Any +from constance import config from django.core.cache import cache - import sentry_sdk from redis_lock import Lock from country_workspace.config.celery import app -from country_workspace.models import AsyncJob +from country_workspace.contrib.kobo.client import Client as KoboClient +from country_workspace.models import AsyncJob, KoboAsset, KoboSubmission +from country_workspace.models.jobs import KoboSyncJob +from country_workspace.models.kobo import KoboQuestion logger = logging.getLogger(__name__) @@ -60,3 +63,16 @@ def sync_job_task(pk: int, version: int) -> dict[str, Any]: @app.task() def removed_expired_jobs(**kwargs): AsyncJob.objects.filter(**kwargs).delete() + +@app.task +def sync_kobo_assets_task(job_id: int, version: int) -> None: + _ = KoboSyncJob.objects.get(pk=job_id, version=version) + client = KoboClient(base_url=config.KOBO_BASE_URL, token=config.KOBO_TOKEN) + for asset_data in client.assets: + asset_model, _ = KoboAsset.objects.update_or_create(uid=asset_data.uid, defaults={"name": asset_data.name}) + + for question_data in asset_data.questions: + KoboQuestion.objects.update_or_create(asset=asset_model, key=question_data.key, labels=question_data.labels) + + for submission_data in asset_data.submissions: + KoboSubmission.objects.update_or_create(uuid=submission_data.uuid, asset=asset_model, data=submission_data.data) diff --git a/tests/contrib/kobo/test_auth.py b/tests/contrib/kobo/test_auth.py new file mode 100644 index 0000000..3060452 --- /dev/null +++ b/tests/contrib/kobo/test_auth.py @@ -0,0 +1,14 @@ +from unittest.mock import Mock + +from requests.models import PreparedRequest + +from country_workspace.contrib.kobo.auth import Auth, AUTHORIZATION, TOKEN + + +def test_token_is_used() -> None: + api_key = "test_api_key" + auth = Auth(api_key) + request = Mock(spec=PreparedRequest) + request.headers = {} + auth(request) + assert request.headers[AUTHORIZATION] == f"{TOKEN} {api_key}" diff --git a/tests/contrib/kobo/test_kobo_client.py b/tests/contrib/kobo/test_kobo_client.py new file mode 100644 index 0000000..a3da54e --- /dev/null +++ b/tests/contrib/kobo/test_kobo_client.py @@ -0,0 +1,47 @@ +from itertools import batched +from typing import Any +from unittest.mock import Mock + +from pytest import raises +from requests.sessions import Session +from requests.exceptions import Timeout + +from country_workspace.contrib.kobo.client import _handle_paginated_response + + +SAMPLE_URL = "https://example.com" + + +def identity(x: Any) -> Any: + return x + + +def test_all_data_is_fetched() -> None: + data = tuple(range(10)) + paged_results = tuple(batched(data, 3)) + urls = tuple(f"{SAMPLE_URL}/{i}" for i in range(len(paged_results))) + next_urls = urls[1:] + (None,) + previous_urls = (None,) + urls[:-1] + responses = tuple( + {"count": len(data), "next": next_url, "previous": prev_url, "results": results} + for results, next_url, prev_url in zip( + paged_results, next_urls, previous_urls, strict=True + ) + ) + session = Mock(spec=Session) + session.get.return_value.json.side_effect = responses + assert ( + tuple( + _handle_paginated_response( + session, urls[0], lambda x: x["results"], identity + ) + ) + == data + ) + + +def test_error_is_propagated() -> None: + session = Mock(spec=Session) + session.get.return_value.raise_for_status.side_effect = Timeout + with raises(Timeout): + tuple(_handle_paginated_response(session, SAMPLE_URL, identity, identity)) diff --git a/tests/extras/testutils/factories/__init__.py b/tests/extras/testutils/factories/__init__.py index 939d1da..b52424b 100644 --- a/tests/extras/testutils/factories/__init__.py +++ b/tests/extras/testutils/factories/__init__.py @@ -10,7 +10,7 @@ from .django_celery_beat import PeriodicTaskFactory # noqa from .household import CountryHouseholdFactory, HouseholdFactory # noqa from .individual import CountryIndividualFactory, IndividualFactory # noqa -from .job import AsyncJobFactory # noqa +from .job import AsyncJobFactory, KoboSyncJobFactory # noqa from .locations import AreaFactory, AreaTypeFactory, CountryFactory # noqa from .office import OfficeFactory # noqa from .program import CountryProgramFactory, ProgramFactory # noqa @@ -21,6 +21,7 @@ from .userrole import UserRole, UserRoleFactory # noqa from .version import ScriptFactory # noqa from .workspaces import CountryChecker # noqa +from .kobo import KoboAssetFactory # noqa for _, name, _ in pkgutil.iter_modules([str(Path(__file__).parent)]): importlib.import_module(f".{name}", __package__) diff --git a/tests/extras/testutils/factories/job.py b/tests/extras/testutils/factories/job.py index 897f51c..a4805ef 100644 --- a/tests/extras/testutils/factories/job.py +++ b/tests/extras/testutils/factories/job.py @@ -1,6 +1,7 @@ import factory from country_workspace.models import AsyncJob +from country_workspace.models.jobs import KoboSyncJob from .base import AutoRegisterModelFactory from .program import ProgramFactory @@ -15,3 +16,8 @@ class AsyncJobFactory(AutoRegisterModelFactory): class Meta: model = AsyncJob + + +class KoboSyncJobFactory(AutoRegisterModelFactory): + class Meta: + model = KoboSyncJob diff --git a/tests/extras/testutils/factories/kobo.py b/tests/extras/testutils/factories/kobo.py new file mode 100644 index 0000000..e592475 --- /dev/null +++ b/tests/extras/testutils/factories/kobo.py @@ -0,0 +1,12 @@ +from factory import fuzzy + +from country_workspace.models import KoboAsset +from testutils.factories import AutoRegisterModelFactory + + +class KoboAssetFactory(AutoRegisterModelFactory): + uid = fuzzy.FuzzyText() + name = fuzzy.FuzzyText() + + class Meta: + model = KoboAsset