From e6816794358bdf7fd89178b1524b67874d2033a2 Mon Sep 17 00:00:00 2001
From: Sergey Misuk <sergey.misuk@gmail.com>
Date: Wed, 23 Oct 2024 01:31:47 +0300
Subject: [PATCH 01/10] Implement Kobo client

---
 src/country_workspace/sync/kobo/__init__.py |  0
 src/country_workspace/sync/kobo/auth.py     | 11 ++++++
 src/country_workspace/sync/kobo/client.py   | 38 +++++++++++++++++++++
 src/country_workspace/sync/kobo/data.py     | 18 ++++++++++
 src/country_workspace/sync/kobo/raw_data.py | 38 +++++++++++++++++++++
 5 files changed, 105 insertions(+)
 create mode 100644 src/country_workspace/sync/kobo/__init__.py
 create mode 100644 src/country_workspace/sync/kobo/auth.py
 create mode 100644 src/country_workspace/sync/kobo/client.py
 create mode 100644 src/country_workspace/sync/kobo/data.py
 create mode 100644 src/country_workspace/sync/kobo/raw_data.py

diff --git a/src/country_workspace/sync/kobo/__init__.py b/src/country_workspace/sync/kobo/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/country_workspace/sync/kobo/auth.py b/src/country_workspace/sync/kobo/auth.py
new file mode 100644
index 0000000..3b6cc47
--- /dev/null
+++ b/src/country_workspace/sync/kobo/auth.py
@@ -0,0 +1,11 @@
+from requests.auth import AuthBase
+from requests.models import PreparedRequest
+
+
+class Auth(AuthBase):
+    def __init__(self, api_key: str) -> None:
+        self._auth_header = f"Token {api_key}"
+
+    def __call__(self, request: PreparedRequest) -> PreparedRequest:
+        request.headers["Authorization"] = self._auth_header
+        return request
diff --git a/src/country_workspace/sync/kobo/client.py b/src/country_workspace/sync/kobo/client.py
new file mode 100644
index 0000000..3cee95b
--- /dev/null
+++ b/src/country_workspace/sync/kobo/client.py
@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from functools import partial
+
+from requests import Session
+
+from country_workspace.sync.kobo.auth import Auth
+from country_workspace.sync.kobo.data import Asset, Datum
+from country_workspace.sync.kobo.raw_data import AssetListResponse, DataResponse
+
+
+class URLs:
+    def __init__(self, base_url) -> None:
+        self._base_url = base_url
+
+    @property
+    def asset_list(self) -> str:
+        return f"{self._base_url}/api/v2/assets.json"
+
+
+class Client:
+    def __init__(self, urls: URLs, auth: Auth) -> None:
+        self.urls = urls
+        self.session = Session()
+        self.session.auth = auth
+
+    @property
+    def assets(self) -> tuple[Asset, ...]:
+        response = self.session.get(self.urls.asset_list)
+        response.raise_for_status()
+        data: AssetListResponse = response.json()
+        return tuple(Asset(raw, partial(self._data, raw["data"])) for raw in data["results"])
+
+    def _data(self, url: str) -> tuple[Datum, ...]:
+        response = self.session.get(url)
+        response.raise_for_status()
+        data: DataResponse = response.json()
+        return tuple(Datum(raw) for raw in data["results"])
diff --git a/src/country_workspace/sync/kobo/data.py b/src/country_workspace/sync/kobo/data.py
new file mode 100644
index 0000000..1e847d8
--- /dev/null
+++ b/src/country_workspace/sync/kobo/data.py
@@ -0,0 +1,18 @@
+from collections.abc import Callable
+
+from country_workspace.sync.kobo.raw_data import AssetListItem, DataItem
+
+
+class Datum:
+    def __init__(self, raw: DataItem) -> None:
+        self._raw = raw
+
+
+class Asset:
+    def __init__(self, raw: AssetListItem, data_thunk: Callable[[], tuple[Datum, ...]]) -> None:
+        self._raw = raw
+        self._data_thunk = data_thunk
+
+    @property
+    def data(self) -> tuple[Datum, ...]:
+        return self._data_thunk()
diff --git a/src/country_workspace/sync/kobo/raw_data.py b/src/country_workspace/sync/kobo/raw_data.py
new file mode 100644
index 0000000..3e35eac
--- /dev/null
+++ b/src/country_workspace/sync/kobo/raw_data.py
@@ -0,0 +1,38 @@
+from typing import TypedDict, Any
+
+
+class Response(TypedDict):
+    count: int
+    next: str | None
+    previous: str | None
+
+
+class AssetListItem(TypedDict):
+    name: str
+    data: str
+
+
+class AssetListResponse(Response):
+    results: list[AssetListItem]
+
+
+class DataItem(TypedDict("AssetDataItem", {"formhub/uuid": str,
+                                                "meta/instanceID": str})):
+    __version__: str
+    _attachments: list
+    _geolocation: tuple[float, float] | tuple[None, None]
+    _id: int
+    _notes: list
+    _status: Any
+    _submission_time: str
+    _submitted_by: str | None
+    _tags: list
+    _uuid: str
+    _validation_status: dict
+    _xform_id_string: str
+    end: str
+    start: str
+
+
+class DataResponse(Response):
+    results: list[DataItem]

From fe387c25cf410e7725f6eeae80ab58c399e9576e Mon Sep 17 00:00:00 2001
From: Sergey Misuk <sergey.misuk@gmail.com>
Date: Thu, 24 Oct 2024 11:51:37 +0300
Subject: [PATCH 02/10] Handle paginated response

---
 src/country_workspace/sync/kobo/client.py | 40 +++++++++++++++--------
 1 file changed, 26 insertions(+), 14 deletions(-)

diff --git a/src/country_workspace/sync/kobo/client.py b/src/country_workspace/sync/kobo/client.py
index 3cee95b..78de508 100644
--- a/src/country_workspace/sync/kobo/client.py
+++ b/src/country_workspace/sync/kobo/client.py
@@ -1,12 +1,12 @@
-from __future__ import annotations
-
+from collections.abc import Generator, Callable
 from functools import partial
+from typing import cast
 
 from requests import Session
 
 from country_workspace.sync.kobo.auth import Auth
 from country_workspace.sync.kobo.data import Asset, Datum
-from country_workspace.sync.kobo.raw_data import AssetListResponse, DataResponse
+from country_workspace.sync.kobo.raw_data import AssetListResponse, DataResponse, Response, AssetListItem
 
 
 class URLs:
@@ -18,6 +18,18 @@ def asset_list(self) -> str:
         return f"{self._base_url}/api/v2/assets.json"
 
 
+def handle_paginated_response[T, U](session: Session,
+                                    url: str,
+                                    collection_mapper: Callable[[Response], list[T]],
+                                    item_mapper: Callable[[T], U]) -> Generator[U, None, None]:
+    while url:
+        response = session.get(url)
+        response.raise_for_status()
+        data: Response = response.json()
+        yield from map(item_mapper, collection_mapper(data))
+        url = data["next"]
+
+
 class Client:
     def __init__(self, urls: URLs, auth: Auth) -> None:
         self.urls = urls
@@ -25,14 +37,14 @@ def __init__(self, urls: URLs, auth: Auth) -> None:
         self.session.auth = auth
 
     @property
-    def assets(self) -> tuple[Asset, ...]:
-        response = self.session.get(self.urls.asset_list)
-        response.raise_for_status()
-        data: AssetListResponse = response.json()
-        return tuple(Asset(raw, partial(self._data, raw["data"])) for raw in data["results"])
-
-    def _data(self, url: str) -> tuple[Datum, ...]:
-        response = self.session.get(url)
-        response.raise_for_status()
-        data: DataResponse = response.json()
-        return tuple(Datum(raw) for raw in data["results"])
+    def assets(self) -> Generator[Asset, None, None]:
+        return handle_paginated_response(self.session,
+                                         self.urls.asset_list,
+                                         lambda r: cast(AssetListResponse, r)["results"],
+                                         lambda i: Asset(i, partial(self._get_asset_data, i)))
+
+    def _get_asset_data(self, raw: AssetListItem) -> Generator[Datum, None, None]:
+        return handle_paginated_response(self.session,
+                                         raw["data"],
+                                         lambda r: cast(DataResponse, r)["results"],
+                                         Datum)

From d3c132c6a9423894de216619034d07c8c600493f Mon Sep 17 00:00:00 2001
From: Sergey Misuk <sergey.misuk@gmail.com>
Date: Wed, 20 Nov 2024 14:52:24 +0300
Subject: [PATCH 03/10] Refactor client code

---
 src/country_workspace/sync/kobo/client.py     | 50 ++++++++---------
 src/country_workspace/sync/kobo/data.py       | 54 +++++++++++++++----
 src/country_workspace/sync/kobo/raw/asset.py  | 15 ++++++
 .../sync/kobo/raw/asset_list.py               | 12 +++++
 src/country_workspace/sync/kobo/raw/common.py |  7 +++
 .../sync/kobo/raw/submission_list.py          | 11 ++++
 src/country_workspace/sync/kobo/raw_data.py   | 38 -------------
 7 files changed, 114 insertions(+), 73 deletions(-)
 create mode 100644 src/country_workspace/sync/kobo/raw/asset.py
 create mode 100644 src/country_workspace/sync/kobo/raw/asset_list.py
 create mode 100644 src/country_workspace/sync/kobo/raw/common.py
 create mode 100644 src/country_workspace/sync/kobo/raw/submission_list.py
 delete mode 100644 src/country_workspace/sync/kobo/raw_data.py

diff --git a/src/country_workspace/sync/kobo/client.py b/src/country_workspace/sync/kobo/client.py
index 78de508..b3f75c0 100644
--- a/src/country_workspace/sync/kobo/client.py
+++ b/src/country_workspace/sync/kobo/client.py
@@ -1,50 +1,50 @@
 from collections.abc import Generator, Callable
-from functools import partial
 from typing import cast
 
+from black.linegen import partial
 from requests import Session
 
 from country_workspace.sync.kobo.auth import Auth
-from country_workspace.sync.kobo.data import Asset, Datum
-from country_workspace.sync.kobo.raw_data import AssetListResponse, DataResponse, Response, AssetListItem
-
-
-class URLs:
-    def __init__(self, base_url) -> None:
-        self._base_url = base_url
-
-    @property
-    def asset_list(self) -> str:
-        return f"{self._base_url}/api/v2/assets.json"
+from country_workspace.sync.kobo.data import Submission, Asset, Question
+from country_workspace.sync.kobo.raw.common import ListResponse
+from country_workspace.sync.kobo.raw import asset as raw_asset
+from country_workspace.sync.kobo.raw import asset_list as raw_asset_list
+from country_workspace.sync.kobo.raw import submission_list as raw_submission_list
 
 
 def handle_paginated_response[T, U](session: Session,
                                     url: str,
-                                    collection_mapper: Callable[[Response], list[T]],
+                                    collection_mapper: Callable[[ListResponse], list[T]],
                                     item_mapper: Callable[[T], U]) -> Generator[U, None, None]:
     while url:
         response = session.get(url)
         response.raise_for_status()
-        data: Response = response.json()
+        data: ListResponse = response.json()
         yield from map(item_mapper, collection_mapper(data))
         url = data["next"]
 
 
 class Client:
-    def __init__(self, urls: URLs, auth: Auth) -> None:
-        self.urls = urls
+    def __init__(self, *, base_url: str, token: str) -> None:
+        self.base_url = base_url
         self.session = Session()
-        self.session.auth = auth
+        self.session.auth = Auth(token)
 
     @property
     def assets(self) -> Generator[Asset, None, None]:
-        return handle_paginated_response(self.session,
-                                         self.urls.asset_list,
-                                         lambda r: cast(AssetListResponse, r)["results"],
-                                         lambda i: Asset(i, partial(self._get_asset_data, i)))
+        yield from handle_paginated_response(self.session,
+                                             f"{self.base_url}/api/v2/assets.json",
+                                             lambda i: cast(raw_asset_list.AssetList, i)["results"],
+                                             self._get_asset_data)
+
+    def _get_asset_data(self, raw: raw_asset_list.Asset) -> Asset:
+        response = self.session.get(raw["url"])
+        response.raise_for_status()
+        data: raw_asset.Asset = response.json()
+        return Asset(data, self._get_asset_submissions(raw["data"]))
 
-    def _get_asset_data(self, raw: AssetListItem) -> Generator[Datum, None, None]:
+    def _get_asset_submissions(self, url: str) -> Generator[Callable[[list[Question]], Submission], None, None]:
         return handle_paginated_response(self.session,
-                                         raw["data"],
-                                         lambda r: cast(DataResponse, r)["results"],
-                                         Datum)
+                                         url,
+                                         lambda i: cast(raw_submission_list.SubmissionList, i)["results"],
+                                         lambda i: partial(Submission, i))
diff --git a/src/country_workspace/sync/kobo/data.py b/src/country_workspace/sync/kobo/data.py
index 1e847d8..06c59c7 100644
--- a/src/country_workspace/sync/kobo/data.py
+++ b/src/country_workspace/sync/kobo/data.py
@@ -1,18 +1,52 @@
-from collections.abc import Callable
+from collections import UserDict
+from collections.abc import Callable, Generator
 
-from country_workspace.sync.kobo.raw_data import AssetListItem, DataItem
+from country_workspace.sync.kobo.raw import asset as raw_asset
+from country_workspace.sync.kobo.raw import submission_list as raw_submission_list
 
 
-class Datum:
-    def __init__(self, raw: DataItem) -> None:
+class Raw[T]:
+    def __init__(self, raw: T) -> None:
         self._raw = raw
 
 
-class Asset:
-    def __init__(self, raw: AssetListItem, data_thunk: Callable[[], tuple[Datum, ...]]) -> None:
-        self._raw = raw
-        self._data_thunk = data_thunk
+class Question(Raw[raw_asset.Question]):
+    @property
+    def key(self) -> str:
+        return self._raw["$autoname"]
+
+    @property
+    def label(self) -> list[str]:
+        return self._raw["label"]
+
+
+class Submission(Raw[raw_submission_list.Submission], UserDict):
+    def __init__(self, raw: raw_submission_list.Submission, questions: list[Question]) -> None:
+        Raw.__init__(self, raw)
+        UserDict.__init__(self, {question.key: raw[question.key] for question in questions})
+
+    @property
+    def id(self) -> int:
+        return self._raw["_id"]
+
+
+class Asset(Raw[raw_asset.Asset]):
+    def __init__(self, raw: raw_asset.Asset, submissions: Generator[Callable[[list[Question]], Submission], None, None]) -> None:
+        super().__init__(raw)
+        self._submissions = submissions
+
+    @property
+    def id(self) -> str:
+        return self._raw["uid"]
+
+    @property
+    def name(self) -> str:
+        return self._raw["name"]
+
+    @property
+    def questions(self) -> list[Question]:
+        return [Question(raw_question) for raw_question in self._raw["content"]["survey"] if "label" in raw_question]
 
     @property
-    def data(self) -> tuple[Datum, ...]:
-        return self._data_thunk()
+    def submissions(self) -> Generator[Submission, None, None]:
+        return (submission(self.questions) for submission in self._submissions)
diff --git a/src/country_workspace/sync/kobo/raw/asset.py b/src/country_workspace/sync/kobo/raw/asset.py
new file mode 100644
index 0000000..4c5d9d7
--- /dev/null
+++ b/src/country_workspace/sync/kobo/raw/asset.py
@@ -0,0 +1,15 @@
+from typing import TypedDict
+
+
+class Question(TypedDict("Question", {"$autoname": str})):
+    label: list[str]
+
+
+class Content(TypedDict):
+    survey: list[Question]
+
+
+class Asset(TypedDict):
+    content: Content
+    name: str
+    uid: str
diff --git a/src/country_workspace/sync/kobo/raw/asset_list.py b/src/country_workspace/sync/kobo/raw/asset_list.py
new file mode 100644
index 0000000..3bf2b21
--- /dev/null
+++ b/src/country_workspace/sync/kobo/raw/asset_list.py
@@ -0,0 +1,12 @@
+from typing import TypedDict
+
+from country_workspace.sync.kobo.raw.common import ListResponse
+
+
+class Asset(TypedDict):
+    data: str
+    url: str
+
+
+class AssetList(ListResponse):
+    results: list[Asset]
diff --git a/src/country_workspace/sync/kobo/raw/common.py b/src/country_workspace/sync/kobo/raw/common.py
new file mode 100644
index 0000000..17f54f6
--- /dev/null
+++ b/src/country_workspace/sync/kobo/raw/common.py
@@ -0,0 +1,7 @@
+from typing import TypedDict
+
+
+class ListResponse(TypedDict):
+    count: int
+    next: str | None
+    previous: str | None
diff --git a/src/country_workspace/sync/kobo/raw/submission_list.py b/src/country_workspace/sync/kobo/raw/submission_list.py
new file mode 100644
index 0000000..7e3b056
--- /dev/null
+++ b/src/country_workspace/sync/kobo/raw/submission_list.py
@@ -0,0 +1,11 @@
+from typing import TypedDict
+
+from country_workspace.sync.kobo.raw.common import ListResponse
+
+
+class Submission(TypedDict):
+    _id: int
+
+
+class SubmissionList(ListResponse):
+    results: list[Submission]
diff --git a/src/country_workspace/sync/kobo/raw_data.py b/src/country_workspace/sync/kobo/raw_data.py
deleted file mode 100644
index 3e35eac..0000000
--- a/src/country_workspace/sync/kobo/raw_data.py
+++ /dev/null
@@ -1,38 +0,0 @@
-from typing import TypedDict, Any
-
-
-class Response(TypedDict):
-    count: int
-    next: str | None
-    previous: str | None
-
-
-class AssetListItem(TypedDict):
-    name: str
-    data: str
-
-
-class AssetListResponse(Response):
-    results: list[AssetListItem]
-
-
-class DataItem(TypedDict("AssetDataItem", {"formhub/uuid": str,
-                                                "meta/instanceID": str})):
-    __version__: str
-    _attachments: list
-    _geolocation: tuple[float, float] | tuple[None, None]
-    _id: int
-    _notes: list
-    _status: Any
-    _submission_time: str
-    _submitted_by: str | None
-    _tags: list
-    _uuid: str
-    _validation_status: dict
-    _xform_id_string: str
-    end: str
-    start: str
-
-
-class DataResponse(Response):
-    results: list[DataItem]

From 47079a0dbc173f4dbdbdee7f0b2b8cf4b4d9f16c Mon Sep 17 00:00:00 2001
From: Sergey Misuk <sergey.misuk@gmail.com>
Date: Fri, 22 Nov 2024 14:43:21 +0300
Subject: [PATCH 04/10] Import data

---
 src/country_workspace/admin/__init__.py       |  1 +
 src/country_workspace/admin/job.py            | 10 +-
 src/country_workspace/admin/kobo.py           | 14 +++
 .../{sync => contrib}/kobo/__init__.py        |  0
 .../{sync => contrib}/kobo/auth.py            |  0
 .../{sync => contrib}/kobo/client.py          | 11 +--
 .../{sync => contrib}/kobo/data.py            | 11 ++-
 .../contrib/kobo/raw/__init__.py              |  0
 .../{sync => contrib}/kobo/raw/asset.py       |  0
 .../{sync => contrib}/kobo/raw/asset_list.py  |  2 +-
 .../{sync => contrib}/kobo/raw/common.py      |  0
 .../kobo/raw/submission_list.py               |  4 +-
 ...02_koboasset_kobosyncjob_kobosubmission.py | 95 +++++++++++++++++++
 src/country_workspace/models/__init__.py      |  3 +-
 src/country_workspace/models/jobs.py          |  8 ++
 src/country_workspace/models/kobo.py          | 11 +++
 src/country_workspace/tasks.py                | 13 ++-
 17 files changed, 164 insertions(+), 19 deletions(-)
 create mode 100644 src/country_workspace/admin/kobo.py
 rename src/country_workspace/{sync => contrib}/kobo/__init__.py (100%)
 rename src/country_workspace/{sync => contrib}/kobo/auth.py (100%)
 rename src/country_workspace/{sync => contrib}/kobo/client.py (82%)
 rename src/country_workspace/{sync => contrib}/kobo/data.py (83%)
 create mode 100644 src/country_workspace/contrib/kobo/raw/__init__.py
 rename src/country_workspace/{sync => contrib}/kobo/raw/asset.py (100%)
 rename src/country_workspace/{sync => contrib}/kobo/raw/asset_list.py (67%)
 rename src/country_workspace/{sync => contrib}/kobo/raw/common.py (100%)
 rename src/country_workspace/{sync => contrib}/kobo/raw/submission_list.py (61%)
 create mode 100644 src/country_workspace/migrations/0002_koboasset_kobosyncjob_kobosubmission.py
 create mode 100644 src/country_workspace/models/kobo.py

diff --git a/src/country_workspace/admin/__init__.py b/src/country_workspace/admin/__init__.py
index 16d872b..e75377a 100644
--- a/src/country_workspace/admin/__init__.py
+++ b/src/country_workspace/admin/__init__.py
@@ -16,6 +16,7 @@
 from .role import UserRoleAdmin  # noqa
 from .sync import SyncLog  # noqa
 from .user import UserAdmin  # noqa
+from .kobo import KoboAsset, KoboSubmission  # noqa
 
 site.register(ContentType, admin_class=ContentTypeAdmin)
 site.register(Permission, admin_class=PermissionAdmin)
diff --git a/src/country_workspace/admin/job.py b/src/country_workspace/admin/job.py
index 43ce2f3..4df856e 100644
--- a/src/country_workspace/admin/job.py
+++ b/src/country_workspace/admin/job.py
@@ -1,11 +1,11 @@
 from typing import TYPE_CHECKING, Sequence
 
 from django.contrib import admin
-
 from adminfilters.autocomplete import AutoCompleteFilter, LinkedAutoCompleteFilter
+from django.http import HttpRequest
 from django_celery_boost.admin import CeleryTaskModelAdmin
 
-from ..models import AsyncJob
+from ..models import AsyncJob, KoboSyncJob
 from .base import BaseModelAdmin
 from .filters import FailedFilter
 
@@ -27,5 +27,9 @@ class AsyncJobAdmin(CeleryTaskModelAdmin, BaseModelAdmin):
 
     def get_readonly_fields(self, request: "HttpRequest", obj: "AsyncJob | None" = None) -> Sequence[str]:
         if obj:
-            return ("program", "batch", "owner", "local_status", "type", "action", "sentry_id")
+            return "program", "batch", "owner", "local_status", "type", "action", "sentry_id"
         return super().get_readonly_fields(request, obj)
+
+@admin.register(KoboSyncJob)
+class DedupJobAdmin(CeleryTaskModelAdmin, BaseModelAdmin):
+    pass
diff --git a/src/country_workspace/admin/kobo.py b/src/country_workspace/admin/kobo.py
new file mode 100644
index 0000000..d495e37
--- /dev/null
+++ b/src/country_workspace/admin/kobo.py
@@ -0,0 +1,14 @@
+from django.contrib import admin
+
+from ..models import KoboAsset, KoboSubmission
+from .base import BaseModelAdmin
+
+
+@admin.register(KoboAsset)
+class KoboAssetAdmin(BaseModelAdmin):
+    pass
+
+
+@admin.register(KoboSubmission)
+class KoboSubmissionAdmin(BaseModelAdmin):
+    pass
diff --git a/src/country_workspace/sync/kobo/__init__.py b/src/country_workspace/contrib/kobo/__init__.py
similarity index 100%
rename from src/country_workspace/sync/kobo/__init__.py
rename to src/country_workspace/contrib/kobo/__init__.py
diff --git a/src/country_workspace/sync/kobo/auth.py b/src/country_workspace/contrib/kobo/auth.py
similarity index 100%
rename from src/country_workspace/sync/kobo/auth.py
rename to src/country_workspace/contrib/kobo/auth.py
diff --git a/src/country_workspace/sync/kobo/client.py b/src/country_workspace/contrib/kobo/client.py
similarity index 82%
rename from src/country_workspace/sync/kobo/client.py
rename to src/country_workspace/contrib/kobo/client.py
index b3f75c0..2bfb152 100644
--- a/src/country_workspace/sync/kobo/client.py
+++ b/src/country_workspace/contrib/kobo/client.py
@@ -4,12 +4,11 @@
 from black.linegen import partial
 from requests import Session
 
-from country_workspace.sync.kobo.auth import Auth
-from country_workspace.sync.kobo.data import Submission, Asset, Question
-from country_workspace.sync.kobo.raw.common import ListResponse
-from country_workspace.sync.kobo.raw import asset as raw_asset
-from country_workspace.sync.kobo.raw import asset_list as raw_asset_list
-from country_workspace.sync.kobo.raw import submission_list as raw_submission_list
+from country_workspace.contrib.kobo.auth import Auth
+from country_workspace.contrib.kobo.data import Submission, Asset, Question
+from country_workspace.contrib.kobo.raw.common import ListResponse
+from country_workspace.contrib.kobo.raw import asset as raw_asset, asset_list as raw_asset_list
+from country_workspace.contrib.kobo.raw import submission_list as raw_submission_list
 
 
 def handle_paginated_response[T, U](session: Session,
diff --git a/src/country_workspace/sync/kobo/data.py b/src/country_workspace/contrib/kobo/data.py
similarity index 83%
rename from src/country_workspace/sync/kobo/data.py
rename to src/country_workspace/contrib/kobo/data.py
index 06c59c7..99e3086 100644
--- a/src/country_workspace/sync/kobo/data.py
+++ b/src/country_workspace/contrib/kobo/data.py
@@ -1,8 +1,9 @@
 from collections import UserDict
 from collections.abc import Callable, Generator
+from uuid import UUID
 
-from country_workspace.sync.kobo.raw import asset as raw_asset
-from country_workspace.sync.kobo.raw import submission_list as raw_submission_list
+from country_workspace.contrib.kobo.raw import asset as raw_asset
+from country_workspace.contrib.kobo.raw import submission_list as raw_submission_list
 
 
 class Raw[T]:
@@ -26,8 +27,8 @@ def __init__(self, raw: raw_submission_list.Submission, questions: list[Question
         UserDict.__init__(self, {question.key: raw[question.key] for question in questions})
 
     @property
-    def id(self) -> int:
-        return self._raw["_id"]
+    def uuid(self) -> UUID:
+        return UUID(self._raw["_uuid"])
 
 
 class Asset(Raw[raw_asset.Asset]):
@@ -36,7 +37,7 @@ def __init__(self, raw: raw_asset.Asset, submissions: Generator[Callable[[list[Q
         self._submissions = submissions
 
     @property
-    def id(self) -> str:
+    def uid(self) -> str:
         return self._raw["uid"]
 
     @property
diff --git a/src/country_workspace/contrib/kobo/raw/__init__.py b/src/country_workspace/contrib/kobo/raw/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/country_workspace/sync/kobo/raw/asset.py b/src/country_workspace/contrib/kobo/raw/asset.py
similarity index 100%
rename from src/country_workspace/sync/kobo/raw/asset.py
rename to src/country_workspace/contrib/kobo/raw/asset.py
diff --git a/src/country_workspace/sync/kobo/raw/asset_list.py b/src/country_workspace/contrib/kobo/raw/asset_list.py
similarity index 67%
rename from src/country_workspace/sync/kobo/raw/asset_list.py
rename to src/country_workspace/contrib/kobo/raw/asset_list.py
index 3bf2b21..5393634 100644
--- a/src/country_workspace/sync/kobo/raw/asset_list.py
+++ b/src/country_workspace/contrib/kobo/raw/asset_list.py
@@ -1,6 +1,6 @@
 from typing import TypedDict
 
-from country_workspace.sync.kobo.raw.common import ListResponse
+from country_workspace.contrib.kobo.raw.common import ListResponse
 
 
 class Asset(TypedDict):
diff --git a/src/country_workspace/sync/kobo/raw/common.py b/src/country_workspace/contrib/kobo/raw/common.py
similarity index 100%
rename from src/country_workspace/sync/kobo/raw/common.py
rename to src/country_workspace/contrib/kobo/raw/common.py
diff --git a/src/country_workspace/sync/kobo/raw/submission_list.py b/src/country_workspace/contrib/kobo/raw/submission_list.py
similarity index 61%
rename from src/country_workspace/sync/kobo/raw/submission_list.py
rename to src/country_workspace/contrib/kobo/raw/submission_list.py
index 7e3b056..9fddda1 100644
--- a/src/country_workspace/sync/kobo/raw/submission_list.py
+++ b/src/country_workspace/contrib/kobo/raw/submission_list.py
@@ -1,10 +1,10 @@
 from typing import TypedDict
 
-from country_workspace.sync.kobo.raw.common import ListResponse
+from country_workspace.contrib.kobo.raw.common import ListResponse
 
 
 class Submission(TypedDict):
-    _id: int
+    _uuid: str
 
 
 class SubmissionList(ListResponse):
diff --git a/src/country_workspace/migrations/0002_koboasset_kobosyncjob_kobosubmission.py b/src/country_workspace/migrations/0002_koboasset_kobosyncjob_kobosubmission.py
new file mode 100644
index 0000000..49dbc10
--- /dev/null
+++ b/src/country_workspace/migrations/0002_koboasset_kobosyncjob_kobosubmission.py
@@ -0,0 +1,95 @@
+# Generated by Django 5.1.2 on 2024-11-21 09:02
+
+import concurrency.fields
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("country_workspace", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="KoboAsset",
+            fields=[
+                ("uid", models.CharField(max_length=32, primary_key=True, serialize=False)),
+            ],
+        ),
+        migrations.CreateModel(
+            name="KoboSyncJob",
+            fields=[
+                ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("version", concurrency.fields.AutoIncVersionField(default=0, help_text="record revision number")),
+                (
+                    "curr_async_result_id",
+                    models.CharField(
+                        blank=True,
+                        editable=False,
+                        help_text="Current (active) AsyncResult is",
+                        max_length=36,
+                        null=True,
+                    ),
+                ),
+                (
+                    "last_async_result_id",
+                    models.CharField(
+                        blank=True, editable=False, help_text="Latest executed AsyncResult is", max_length=36, null=True
+                    ),
+                ),
+                ("datetime_created", models.DateTimeField(auto_now_add=True, help_text="Creation date and time")),
+                (
+                    "datetime_queued",
+                    models.DateTimeField(
+                        blank=True, help_text="Queueing date and time", null=True, verbose_name="Queued At"
+                    ),
+                ),
+                (
+                    "repeatable",
+                    models.BooleanField(
+                        blank=True, default=False, help_text="Indicate if the job can be repeated as-is"
+                    ),
+                ),
+                ("celery_history", models.JSONField(blank=True, default=dict, editable=False)),
+                ("local_status", models.CharField(blank=True, default="", editable=False, max_length=100, null=True)),
+                (
+                    "group_key",
+                    models.CharField(
+                        blank=True,
+                        editable=False,
+                        help_text="Tasks with the same group key will not run in parallel",
+                        max_length=255,
+                        null=True,
+                    ),
+                ),
+                (
+                    "owner",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="%(app_label)s_%(class)s_jobs",
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+                "default_permissions": ("add", "change", "delete", "view", "queue", "terminate", "inspect", "revoke"),
+            },
+        ),
+        migrations.CreateModel(
+            name="KoboSubmission",
+            fields=[
+                ("uuid", models.UUIDField(primary_key=True, serialize=False)),
+                ("data", models.JSONField()),
+                (
+                    "asset",
+                    models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="country_workspace.koboasset"),
+                ),
+            ],
+        ),
+    ]
diff --git a/src/country_workspace/models/__init__.py b/src/country_workspace/models/__init__.py
index f0748e0..5b1f438 100644
--- a/src/country_workspace/models/__init__.py
+++ b/src/country_workspace/models/__init__.py
@@ -1,7 +1,7 @@
 from .batch import Batch  # noqa
 from .household import Household  # noqa
 from .individual import Individual  # noqa
-from .jobs import AsyncJob  # noqa
+from .jobs import AsyncJob, KoboSyncJob  # noqa
 from .locations import Area, AreaType, Country  # noqa
 from .office import Office  # noqa
 from .program import Program  # noqa
@@ -9,3 +9,4 @@
 from .role import UserRole  # noqa
 from .sync import SyncLog  # noqa
 from .user import User  # noqa
+from .kobo import KoboAsset, KoboSubmission  # noqa
diff --git a/src/country_workspace/models/jobs.py b/src/country_workspace/models/jobs.py
index 03d7cf0..2c08b6e 100644
--- a/src/country_workspace/models/jobs.py
+++ b/src/country_workspace/models/jobs.py
@@ -1,6 +1,7 @@
 from typing import Any, Callable
 
 from django.apps import apps
+from django.core.exceptions import ValidationError
 from django.db import models
 from django.utils.module_loading import import_string
 
@@ -61,3 +62,10 @@ def execute(self) -> Any:
             if sid:
                 self.sentry_id = sid
                 self.save(update_fields=["sentry_id"])
+
+class KoboSyncJob(CeleryTaskModel):
+    celery_task_name = "country_workspace.tasks.sync_kobo_assets"
+
+    def clean(self) -> None:
+        if self.__class__.objects.exists() and not self.pk:
+            raise ValidationError(f"You can have only one {self.__class__.__name__} instance.")
diff --git a/src/country_workspace/models/kobo.py b/src/country_workspace/models/kobo.py
new file mode 100644
index 0000000..d3eada1
--- /dev/null
+++ b/src/country_workspace/models/kobo.py
@@ -0,0 +1,11 @@
+from django.db import models
+
+
+class KoboAsset(models.Model):
+    uid = models.CharField(primary_key=True, max_length=32)
+
+
+class KoboSubmission(models.Model):
+    uuid = models.UUIDField(primary_key=True)
+    asset = models.ForeignKey(KoboAsset, on_delete=models.CASCADE)
+    data = models.JSONField()
diff --git a/src/country_workspace/tasks.py b/src/country_workspace/tasks.py
index aa799b0..a798239 100644
--- a/src/country_workspace/tasks.py
+++ b/src/country_workspace/tasks.py
@@ -8,7 +8,9 @@
 from redis_lock import Lock
 
 from country_workspace.config.celery import app
-from country_workspace.models import AsyncJob
+from country_workspace.contrib.kobo.client import Client as KoboClient
+from country_workspace.models import AsyncJob, KoboAsset, KoboSubmission
+from country_workspace.models.jobs import KoboSyncJob
 
 logger = logging.getLogger(__name__)
 
@@ -61,3 +63,12 @@ def sync_job_task(pk: int, version: int) -> dict[str, Any]:
 @app.task()
 def removed_expired_jobs(**kwargs: Any) -> None:
     AsyncJob.objects.filter(**kwargs).delete()
+
+@app.task
+def sync_kobo_assets(job_id: int, version: int) -> None:
+    _ = KoboSyncJob.objects.get(pk=job_id, version=version)
+    client = KoboClient(base_url="https://kf-hope-stg.unitst.org", token="01f1a122ddad12d7e72f3b86e9d8a637c917bee8")
+    for asset_data in client.assets:
+        asset_model = KoboAsset.objects.get_or_create(uid=asset_data.uid)
+        for submission_data in asset_data.submissions:
+            KoboSubmission.objects.get_or_create(uuid=submission_data.uuid, asset=asset_model, data=submission_data.data)

From 37d18b8c99e6d93a8ee1600efa8fe91b6a978ec1 Mon Sep 17 00:00:00 2001
From: Sergey Misuk <sergey.misuk@gmail.com>
Date: Wed, 27 Nov 2024 12:59:18 +0300
Subject: [PATCH 05/10] Configure admin

---
 src/country_workspace/admin/__init__.py       |  2 +-
 src/country_workspace/admin/job.py            |  2 +-
 src/country_workspace/admin/kobo.py           | 26 +++++++---
 src/country_workspace/admin/program.py        |  9 +++-
 src/country_workspace/contrib/kobo/auth.py    |  7 ++-
 src/country_workspace/contrib/kobo/data.py    |  2 +-
 .../migrations/0003_koboasset_programs.py     | 18 +++++++
 .../migrations/0004_koboasset_name.py         | 18 +++++++
 .../migrations/0005_alter_koboasset_uid.py    | 18 +++++++
 ..._alter_kobosubmission_data_koboquestion.py | 31 ++++++++++++
 ...0007_alter_koboquestion_unique_together.py | 17 +++++++
 src/country_workspace/models/__init__.py      |  2 +-
 src/country_workspace/models/jobs.py          |  2 +-
 src/country_workspace/models/kobo.py          | 21 ++++++++-
 tests/contrib/kobo/test_auth.py               | 14 ++++++
 tests/contrib/kobo/test_client.py             | 47 +++++++++++++++++++
 tests/extras/testutils/factories/__init__.py  |  3 +-
 tests/extras/testutils/factories/job.py       |  6 +++
 tests/extras/testutils/factories/kobo.py      | 10 ++++
 19 files changed, 237 insertions(+), 18 deletions(-)
 create mode 100644 src/country_workspace/migrations/0003_koboasset_programs.py
 create mode 100644 src/country_workspace/migrations/0004_koboasset_name.py
 create mode 100644 src/country_workspace/migrations/0005_alter_koboasset_uid.py
 create mode 100644 src/country_workspace/migrations/0006_alter_kobosubmission_data_koboquestion.py
 create mode 100644 src/country_workspace/migrations/0007_alter_koboquestion_unique_together.py
 create mode 100644 tests/contrib/kobo/test_auth.py
 create mode 100644 tests/contrib/kobo/test_client.py
 create mode 100644 tests/extras/testutils/factories/kobo.py

diff --git a/src/country_workspace/admin/__init__.py b/src/country_workspace/admin/__init__.py
index e75377a..8176789 100644
--- a/src/country_workspace/admin/__init__.py
+++ b/src/country_workspace/admin/__init__.py
@@ -16,7 +16,7 @@
 from .role import UserRoleAdmin  # noqa
 from .sync import SyncLog  # noqa
 from .user import UserAdmin  # noqa
-from .kobo import KoboAsset, KoboSubmission  # noqa
+from .kobo import KoboAsset  # noqa
 
 site.register(ContentType, admin_class=ContentTypeAdmin)
 site.register(Permission, admin_class=PermissionAdmin)
diff --git a/src/country_workspace/admin/job.py b/src/country_workspace/admin/job.py
index 4df856e..1a14e4f 100644
--- a/src/country_workspace/admin/job.py
+++ b/src/country_workspace/admin/job.py
@@ -31,5 +31,5 @@ def get_readonly_fields(self, request: "HttpRequest", obj: "AsyncJob | None" = N
         return super().get_readonly_fields(request, obj)
 
 @admin.register(KoboSyncJob)
-class DedupJobAdmin(CeleryTaskModelAdmin, BaseModelAdmin):
+class KoboSyncJobAdmin(CeleryTaskModelAdmin, BaseModelAdmin):
     pass
diff --git a/src/country_workspace/admin/kobo.py b/src/country_workspace/admin/kobo.py
index d495e37..6f8f591 100644
--- a/src/country_workspace/admin/kobo.py
+++ b/src/country_workspace/admin/kobo.py
@@ -1,14 +1,26 @@
 from django.contrib import admin
 
-from ..models import KoboAsset, KoboSubmission
 from .base import BaseModelAdmin
+from ..models import KoboAsset
+from ..models.kobo import KoboQuestion, KoboSubmission
 
 
-@admin.register(KoboAsset)
-class KoboAssetAdmin(BaseModelAdmin):
-    pass
+class ReadOnlyInlineAdmin(admin.TabularInline):
+    can_create = False
+    can_change = False
+    can_delete = False
+    extra = 0
+
+class KoboQuestionAdmin(ReadOnlyInlineAdmin):
+    model = KoboQuestion
+
 
+class KoboSubmissionAdmin(ReadOnlyInlineAdmin):
+    model = KoboSubmission
 
-@admin.register(KoboSubmission)
-class KoboSubmissionAdmin(BaseModelAdmin):
-    pass
+
+@admin.register(KoboAsset)
+class KoboAssetAdmin(BaseModelAdmin):
+    list_display = ("uid", "name")
+    exclude = ("programs",)
+    inlines = (KoboQuestionAdmin, KoboSubmissionAdmin)
diff --git a/src/country_workspace/admin/program.py b/src/country_workspace/admin/program.py
index ac76d89..bf84b54 100644
--- a/src/country_workspace/admin/program.py
+++ b/src/country_workspace/admin/program.py
@@ -9,13 +9,18 @@
 
 from ..cache.manager import cache_manager
 from ..compat.admin_extra_buttons import confirm_action
-from ..models import Program
+from ..models import Program, KoboAsset
 from .base import BaseModelAdmin
 
 if TYPE_CHECKING:
     from admin_extra_buttons.buttons import LinkButton
 
 
+class KoboAssetInline(admin.TabularInline):
+    model = KoboAsset.programs.through
+    extra = 1
+
+
 @admin.register(Program)
 class ProgramAdmin(BaseModelAdmin):
     list_display = ("name", "sector", "status", "active")
@@ -23,12 +28,14 @@ class ProgramAdmin(BaseModelAdmin):
     list_filter = (("country_office", AutoCompleteFilter), "status", "active", "sector")
     ordering = ("name",)
     autocomplete_fields = ("country_office",)
+    inlines = (KoboAssetInline,)
 
     @button()
     def invalidate_cache(self, request: HttpRequest, pk: str) -> None:
         obj: [Program] = Program.objects.select_related("country_office").get(pk=pk)
         cache_manager.incr_cache_version(program=obj)
 
+
     @link(change_list=False)
     def view_in_workspace(self, btn: "LinkButton") -> None:
         obj = btn.context["original"]
diff --git a/src/country_workspace/contrib/kobo/auth.py b/src/country_workspace/contrib/kobo/auth.py
index 3b6cc47..aec0e44 100644
--- a/src/country_workspace/contrib/kobo/auth.py
+++ b/src/country_workspace/contrib/kobo/auth.py
@@ -2,10 +2,13 @@
 from requests.models import PreparedRequest
 
 
+TOKEN = "Token"
+AUTHORIZATION = 'Authorization'
+
 class Auth(AuthBase):
     def __init__(self, api_key: str) -> None:
-        self._auth_header = f"Token {api_key}"
+        self._auth_header = f"{TOKEN} {api_key}"
 
     def __call__(self, request: PreparedRequest) -> PreparedRequest:
-        request.headers["Authorization"] = self._auth_header
+        request.headers[AUTHORIZATION] = self._auth_header
         return request
diff --git a/src/country_workspace/contrib/kobo/data.py b/src/country_workspace/contrib/kobo/data.py
index 99e3086..0316e47 100644
--- a/src/country_workspace/contrib/kobo/data.py
+++ b/src/country_workspace/contrib/kobo/data.py
@@ -17,7 +17,7 @@ def key(self) -> str:
         return self._raw["$autoname"]
 
     @property
-    def label(self) -> list[str]:
+    def labels(self) -> list[str]:
         return self._raw["label"]
 
 
diff --git a/src/country_workspace/migrations/0003_koboasset_programs.py b/src/country_workspace/migrations/0003_koboasset_programs.py
new file mode 100644
index 0000000..54836fb
--- /dev/null
+++ b/src/country_workspace/migrations/0003_koboasset_programs.py
@@ -0,0 +1,18 @@
+# Generated by Django 5.1.2 on 2024-11-25 09:25
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("country_workspace", "0002_koboasset_kobosyncjob_kobosubmission"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="koboasset",
+            name="programs",
+            field=models.ManyToManyField(to="country_workspace.program"),
+        ),
+    ]
diff --git a/src/country_workspace/migrations/0004_koboasset_name.py b/src/country_workspace/migrations/0004_koboasset_name.py
new file mode 100644
index 0000000..cc3f604
--- /dev/null
+++ b/src/country_workspace/migrations/0004_koboasset_name.py
@@ -0,0 +1,18 @@
+# Generated by Django 5.1.2 on 2024-11-25 20:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("country_workspace", "0003_koboasset_programs"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="koboasset",
+            name="name",
+            field=models.CharField(editable=False, max_length=128, null=True),
+        ),
+    ]
diff --git a/src/country_workspace/migrations/0005_alter_koboasset_uid.py b/src/country_workspace/migrations/0005_alter_koboasset_uid.py
new file mode 100644
index 0000000..ef8844a
--- /dev/null
+++ b/src/country_workspace/migrations/0005_alter_koboasset_uid.py
@@ -0,0 +1,18 @@
+# Generated by Django 5.1.2 on 2024-11-25 20:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("country_workspace", "0004_koboasset_name"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="koboasset",
+            name="uid",
+            field=models.CharField(editable=False, max_length=32, primary_key=True, serialize=False),
+        ),
+    ]
diff --git a/src/country_workspace/migrations/0006_alter_kobosubmission_data_koboquestion.py b/src/country_workspace/migrations/0006_alter_kobosubmission_data_koboquestion.py
new file mode 100644
index 0000000..175ac4e
--- /dev/null
+++ b/src/country_workspace/migrations/0006_alter_kobosubmission_data_koboquestion.py
@@ -0,0 +1,31 @@
+# Generated by Django 5.1.2 on 2024-11-25 20:41
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("country_workspace", "0005_alter_koboasset_uid"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="kobosubmission",
+            name="data",
+            field=models.JSONField(default=dict),
+        ),
+        migrations.CreateModel(
+            name="KoboQuestion",
+            fields=[
+                ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("key", models.CharField(editable=False, max_length=128, null=True)),
+                ("labels", models.JSONField(default=list)),
+                (
+                    "asset",
+                    models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="country_workspace.koboasset"),
+                ),
+            ],
+        ),
+    ]
diff --git a/src/country_workspace/migrations/0007_alter_koboquestion_unique_together.py b/src/country_workspace/migrations/0007_alter_koboquestion_unique_together.py
new file mode 100644
index 0000000..241acc1
--- /dev/null
+++ b/src/country_workspace/migrations/0007_alter_koboquestion_unique_together.py
@@ -0,0 +1,17 @@
+# Generated by Django 5.1.2 on 2024-11-25 21:54
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("country_workspace", "0006_alter_kobosubmission_data_koboquestion"),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name="koboquestion",
+            unique_together={("asset", "key")},
+        ),
+    ]
diff --git a/src/country_workspace/models/__init__.py b/src/country_workspace/models/__init__.py
index 5b1f438..2e01086 100644
--- a/src/country_workspace/models/__init__.py
+++ b/src/country_workspace/models/__init__.py
@@ -9,4 +9,4 @@
 from .role import UserRole  # noqa
 from .sync import SyncLog  # noqa
 from .user import User  # noqa
-from .kobo import KoboAsset, KoboSubmission  # noqa
+from .kobo import KoboAsset, KoboSubmission, KoboQuestion  # noqa
diff --git a/src/country_workspace/models/jobs.py b/src/country_workspace/models/jobs.py
index 2c08b6e..e3c031b 100644
--- a/src/country_workspace/models/jobs.py
+++ b/src/country_workspace/models/jobs.py
@@ -64,7 +64,7 @@ def execute(self) -> Any:
                 self.save(update_fields=["sentry_id"])
 
 class KoboSyncJob(CeleryTaskModel):
-    celery_task_name = "country_workspace.tasks.sync_kobo_assets"
+    celery_task_name = "country_workspace.tasks.sync_kobo_assets_task"
 
     def clean(self) -> None:
         if self.__class__.objects.exists() and not self.pk:
diff --git a/src/country_workspace/models/kobo.py b/src/country_workspace/models/kobo.py
index d3eada1..fe291c5 100644
--- a/src/country_workspace/models/kobo.py
+++ b/src/country_workspace/models/kobo.py
@@ -1,11 +1,28 @@
 from django.db import models
 
+from country_workspace.models import Program
+
 
 class KoboAsset(models.Model):
-    uid = models.CharField(primary_key=True, max_length=32)
+    uid = models.CharField(primary_key=True, max_length=32, editable=False)
+    name = models.CharField(max_length=128, null=True, editable=False)
+    programs = models.ManyToManyField(Program)
+
+    def __str__(self) -> str:
+        return self.name or "No name"
+
+
+
+class KoboQuestion(models.Model):
+    asset = models.ForeignKey(KoboAsset, on_delete=models.CASCADE)
+    key = models.CharField(max_length=128, null=True, editable=False)
+    labels = models.JSONField(default=list)
+
+    class Meta:
+        unique_together = ("asset", "key")
 
 
 class KoboSubmission(models.Model):
     uuid = models.UUIDField(primary_key=True)
     asset = models.ForeignKey(KoboAsset, on_delete=models.CASCADE)
-    data = models.JSONField()
+    data = models.JSONField(default=dict)
diff --git a/tests/contrib/kobo/test_auth.py b/tests/contrib/kobo/test_auth.py
new file mode 100644
index 0000000..3060452
--- /dev/null
+++ b/tests/contrib/kobo/test_auth.py
@@ -0,0 +1,14 @@
+from unittest.mock import Mock
+
+from requests.models import PreparedRequest
+
+from country_workspace.contrib.kobo.auth import Auth, AUTHORIZATION, TOKEN
+
+
+def test_token_is_used() -> None:
+    api_key = "test_api_key"
+    auth = Auth(api_key)
+    request = Mock(spec=PreparedRequest)
+    request.headers = {}
+    auth(request)
+    assert request.headers[AUTHORIZATION] == f"{TOKEN} {api_key}"
diff --git a/tests/contrib/kobo/test_client.py b/tests/contrib/kobo/test_client.py
new file mode 100644
index 0000000..901f390
--- /dev/null
+++ b/tests/contrib/kobo/test_client.py
@@ -0,0 +1,47 @@
+from itertools import batched
+from typing import Any
+from unittest.mock import Mock
+
+from pytest import raises
+from requests.sessions import Session
+from requests.exceptions import Timeout
+
+from country_workspace.contrib.kobo.client import handle_paginated_response
+
+
+SAMPLE_URL = "https://example.com"
+
+
+def identity(x: Any) -> Any:
+    return x
+
+
+def test_all_data_is_fetched() -> None:
+    data = tuple(range(10))
+    paged_results = tuple(batched(data, 3))
+    urls = tuple(f"{SAMPLE_URL}/{i}" for i in range(len(paged_results)))
+    next_urls = urls[1:] + (None,)
+    previous_urls = (None,) + urls[:-1]
+    responses = tuple(
+        {"count": len(data), "next": next_url, "previous": prev_url, "results": results}
+        for results, next_url, prev_url in zip(
+            paged_results, next_urls, previous_urls, strict=True
+        )
+    )
+    session = Mock(spec=Session)
+    session.get.return_value.json.side_effect = responses
+    assert (
+        tuple(
+            handle_paginated_response(
+                session, urls[0], lambda x: x["results"], identity
+            )
+        )
+        == data
+    )
+
+
+def test_error_is_propagated() -> None:
+    session = Mock(spec=Session)
+    session.get.return_value.raise_for_status.side_effect = Timeout
+    with raises(Timeout):
+        handle_paginated_response(session, SAMPLE_URL, identity, identity)
diff --git a/tests/extras/testutils/factories/__init__.py b/tests/extras/testutils/factories/__init__.py
index 939d1da..b52424b 100644
--- a/tests/extras/testutils/factories/__init__.py
+++ b/tests/extras/testutils/factories/__init__.py
@@ -10,7 +10,7 @@
 from .django_celery_beat import PeriodicTaskFactory  # noqa
 from .household import CountryHouseholdFactory, HouseholdFactory  # noqa
 from .individual import CountryIndividualFactory, IndividualFactory  # noqa
-from .job import AsyncJobFactory  # noqa
+from .job import AsyncJobFactory, KoboSyncJobFactory  # noqa
 from .locations import AreaFactory, AreaTypeFactory, CountryFactory  # noqa
 from .office import OfficeFactory  # noqa
 from .program import CountryProgramFactory, ProgramFactory  # noqa
@@ -21,6 +21,7 @@
 from .userrole import UserRole, UserRoleFactory  # noqa
 from .version import ScriptFactory  # noqa
 from .workspaces import CountryChecker  # noqa
+from .kobo import KoboAssetFactory  # noqa
 
 for _, name, _ in pkgutil.iter_modules([str(Path(__file__).parent)]):
     importlib.import_module(f".{name}", __package__)
diff --git a/tests/extras/testutils/factories/job.py b/tests/extras/testutils/factories/job.py
index 897f51c..a4805ef 100644
--- a/tests/extras/testutils/factories/job.py
+++ b/tests/extras/testutils/factories/job.py
@@ -1,6 +1,7 @@
 import factory
 
 from country_workspace.models import AsyncJob
+from country_workspace.models.jobs import KoboSyncJob
 
 from .base import AutoRegisterModelFactory
 from .program import ProgramFactory
@@ -15,3 +16,8 @@ class AsyncJobFactory(AutoRegisterModelFactory):
 
     class Meta:
         model = AsyncJob
+
+
+class KoboSyncJobFactory(AutoRegisterModelFactory):
+    class Meta:
+        model = KoboSyncJob
diff --git a/tests/extras/testutils/factories/kobo.py b/tests/extras/testutils/factories/kobo.py
new file mode 100644
index 0000000..4df6239
--- /dev/null
+++ b/tests/extras/testutils/factories/kobo.py
@@ -0,0 +1,10 @@
+from factory import fuzzy
+from factory.declarations import SubFactory
+
+from testutils.factories import AutoRegisterModelFactory, ProgramFactory
+
+
+class KoboAssetFactory(AutoRegisterModelFactory):
+    uid = fuzzy.FuzzyText()
+    name = fuzzy.FuzzyText()
+    program = SubFactory(ProgramFactory)

From 6a8bb967ae02404d486f9a7753698da394bea6a3 Mon Sep 17 00:00:00 2001
From: Sergey Misuk <sergey.misuk@gmail.com>
Date: Thu, 28 Nov 2024 10:11:02 +0300
Subject: [PATCH 06/10] Fix issues

---
 src/country_workspace/admin/kobo.py      | 11 +++++++++++
 src/country_workspace/tasks.py           | 11 ++++++++---
 tests/contrib/kobo/test_client.py        |  2 +-
 tests/extras/testutils/factories/kobo.py |  8 +++++---
 4 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/src/country_workspace/admin/kobo.py b/src/country_workspace/admin/kobo.py
index 6f8f591..db9fdca 100644
--- a/src/country_workspace/admin/kobo.py
+++ b/src/country_workspace/admin/kobo.py
@@ -1,3 +1,5 @@
+from typing import Any
+
 from django.contrib import admin
 
 from .base import BaseModelAdmin
@@ -24,3 +26,12 @@ class KoboAssetAdmin(BaseModelAdmin):
     list_display = ("uid", "name")
     exclude = ("programs",)
     inlines = (KoboQuestionAdmin, KoboSubmissionAdmin)
+
+    def has_add_permission(self, *args: Any, **kwargs: Any) -> bool:
+        return False
+
+    def has_change_permission(self, *args: Any, **kwargs: Any) -> bool:
+        return False
+
+    def has_delete_permission(self, *args: Any, **kwargs: Any) -> bool:
+        return False
diff --git a/src/country_workspace/tasks.py b/src/country_workspace/tasks.py
index a798239..442a13a 100644
--- a/src/country_workspace/tasks.py
+++ b/src/country_workspace/tasks.py
@@ -11,6 +11,7 @@
 from country_workspace.contrib.kobo.client import Client as KoboClient
 from country_workspace.models import AsyncJob, KoboAsset, KoboSubmission
 from country_workspace.models.jobs import KoboSyncJob
+from country_workspace.models.kobo import KoboQuestion
 
 logger = logging.getLogger(__name__)
 
@@ -65,10 +66,14 @@ def removed_expired_jobs(**kwargs: Any) -> None:
     AsyncJob.objects.filter(**kwargs).delete()
 
 @app.task
-def sync_kobo_assets(job_id: int, version: int) -> None:
+def sync_kobo_assets_task(job_id: int, version: int) -> None:
     _ = KoboSyncJob.objects.get(pk=job_id, version=version)
     client = KoboClient(base_url="https://kf-hope-stg.unitst.org", token="01f1a122ddad12d7e72f3b86e9d8a637c917bee8")
     for asset_data in client.assets:
-        asset_model = KoboAsset.objects.get_or_create(uid=asset_data.uid)
+        asset_model, _ = KoboAsset.objects.update_or_create(uid=asset_data.uid, defaults={"name": asset_data.name})
+
+        for question_data in asset_data.questions:
+            KoboQuestion.objects.update_or_create(asset=asset_model, key=question_data.key, labels=question_data.labels)
+
         for submission_data in asset_data.submissions:
-            KoboSubmission.objects.get_or_create(uuid=submission_data.uuid, asset=asset_model, data=submission_data.data)
+            KoboSubmission.objects.update_or_create(uuid=submission_data.uuid, asset=asset_model, data=submission_data.data)
diff --git a/tests/contrib/kobo/test_client.py b/tests/contrib/kobo/test_client.py
index 901f390..8a626d4 100644
--- a/tests/contrib/kobo/test_client.py
+++ b/tests/contrib/kobo/test_client.py
@@ -44,4 +44,4 @@ def test_error_is_propagated() -> None:
     session = Mock(spec=Session)
     session.get.return_value.raise_for_status.side_effect = Timeout
     with raises(Timeout):
-        handle_paginated_response(session, SAMPLE_URL, identity, identity)
+        tuple(handle_paginated_response(session, SAMPLE_URL, identity, identity))
diff --git a/tests/extras/testutils/factories/kobo.py b/tests/extras/testutils/factories/kobo.py
index 4df6239..e592475 100644
--- a/tests/extras/testutils/factories/kobo.py
+++ b/tests/extras/testutils/factories/kobo.py
@@ -1,10 +1,12 @@
 from factory import fuzzy
-from factory.declarations import SubFactory
 
-from testutils.factories import AutoRegisterModelFactory, ProgramFactory
+from country_workspace.models import KoboAsset
+from testutils.factories import AutoRegisterModelFactory
 
 
 class KoboAssetFactory(AutoRegisterModelFactory):
     uid = fuzzy.FuzzyText()
     name = fuzzy.FuzzyText()
-    program = SubFactory(ProgramFactory)
+
+    class Meta:
+        model = KoboAsset

From 8288c737560ac3af8507f73ac9c1195f51d6fedd Mon Sep 17 00:00:00 2001
From: Sergey Misuk <sergey.misuk@gmail.com>
Date: Mon, 2 Dec 2024 16:01:47 +0300
Subject: [PATCH 07/10] Fix test issue

---
 tests/contrib/kobo/{test_client.py => test_kobo_client.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename tests/contrib/kobo/{test_client.py => test_kobo_client.py} (100%)

diff --git a/tests/contrib/kobo/test_client.py b/tests/contrib/kobo/test_kobo_client.py
similarity index 100%
rename from tests/contrib/kobo/test_client.py
rename to tests/contrib/kobo/test_kobo_client.py

From ac88257b8510933f8ded22ce321e39acf0f25ef4 Mon Sep 17 00:00:00 2001
From: Sergey Misuk <sergey.misuk@gmail.com>
Date: Tue, 3 Dec 2024 10:12:23 +0300
Subject: [PATCH 08/10] Take Kobo configuration from environment

---
 src/country_workspace/config/__init__.py       | 2 ++
 src/country_workspace/config/fragments/kobo.py | 4 ++++
 src/country_workspace/config/settings.py       | 1 +
 src/country_workspace/tasks.py                 | 4 ++--
 4 files changed, 9 insertions(+), 2 deletions(-)
 create mode 100644 src/country_workspace/config/fragments/kobo.py

diff --git a/src/country_workspace/config/__init__.py b/src/country_workspace/config/__init__.py
index 3898e59..c663d27 100644
--- a/src/country_workspace/config/__init__.py
+++ b/src/country_workspace/config/__init__.py
@@ -185,6 +185,8 @@ class Group(Enum):
     "AZURE_CLIENT_SECRET": (str, "", "", False, "Azure client secret for SSO"),
     "AZURE_TENANT_ID": (str, "", "", False, "Azure tenant ID for SSO"),
     "AZURE_CLIENT_KEY": (str, "", "", False, "Azure client key for SSO"),
+    "KOBO_BASE_URL": (str, "", "", False, "Kobo API base URL"),
+    "KOBO_TOKEN": (str, "", "", False, "Kobo API token"),
 }
 
 env = SmartEnv(**CONFIG)
diff --git a/src/country_workspace/config/fragments/kobo.py b/src/country_workspace/config/fragments/kobo.py
new file mode 100644
index 0000000..8ecc14b
--- /dev/null
+++ b/src/country_workspace/config/fragments/kobo.py
@@ -0,0 +1,4 @@
+from .. import env
+
+KOBO_BASE_URL = env("KOBO_BASE_URL")
+KOBO_TOKEN = env("KOBO_TOKEN")
diff --git a/src/country_workspace/config/settings.py b/src/country_workspace/config/settings.py
index 1278dde..15b227b 100644
--- a/src/country_workspace/config/settings.py
+++ b/src/country_workspace/config/settings.py
@@ -237,6 +237,7 @@
 from .fragments.rest_framework import *  # noqa
 from .fragments.root import *  # noqa
 from .fragments.sentry import *  # noqa
+from .fragments.kobo import *  # noqa
 
 # from .fragments.smart_admin import *  # noqa
 from .fragments.social_auth import *  # noqa
diff --git a/src/country_workspace/tasks.py b/src/country_workspace/tasks.py
index 442a13a..6c91cfe 100644
--- a/src/country_workspace/tasks.py
+++ b/src/country_workspace/tasks.py
@@ -2,8 +2,8 @@
 import logging
 from typing import TYPE_CHECKING, Any
 
+from constance import config
 from django.core.cache import cache
-
 import sentry_sdk
 from redis_lock import Lock
 
@@ -68,7 +68,7 @@ def removed_expired_jobs(**kwargs: Any) -> None:
 @app.task
 def sync_kobo_assets_task(job_id: int, version: int) -> None:
     _ = KoboSyncJob.objects.get(pk=job_id, version=version)
-    client = KoboClient(base_url="https://kf-hope-stg.unitst.org", token="01f1a122ddad12d7e72f3b86e9d8a637c917bee8")
+    client = KoboClient(base_url=config.KOBO_BASE_URL, token=config.KOBO_TOKEN)
     for asset_data in client.assets:
         asset_model, _ = KoboAsset.objects.update_or_create(uid=asset_data.uid, defaults={"name": asset_data.name})
 

From 0b6f61e9e92c5092be3679babeed07fd9e82ee7e Mon Sep 17 00:00:00 2001
From: Sergey Misuk <sergey.misuk@gmail.com>
Date: Thu, 19 Dec 2024 08:20:40 +0300
Subject: [PATCH 09/10] Handle groups, rosters, and attachments

---
 src/country_workspace/contrib/kobo/client.py  | 77 ++++++++++-----
 src/country_workspace/contrib/kobo/data.py    | 97 +++++++++++++++++--
 .../contrib/kobo/raw/asset.py                 |  7 +-
 .../contrib/kobo/raw/asset_list.py            |  1 +
 .../contrib/kobo/raw/submission_list.py       |  7 ++
 tests/contrib/kobo/test_kobo_client.py        |  6 +-
 6 files changed, 155 insertions(+), 40 deletions(-)

diff --git a/src/country_workspace/contrib/kobo/client.py b/src/country_workspace/contrib/kobo/client.py
index 2bfb152..abcb32f 100644
--- a/src/country_workspace/contrib/kobo/client.py
+++ b/src/country_workspace/contrib/kobo/client.py
@@ -2,48 +2,75 @@
 from typing import cast
 
 from black.linegen import partial
-from requests import Session
+from requests import Session, Response
 
 from country_workspace.contrib.kobo.auth import Auth
 from country_workspace.contrib.kobo.data import Submission, Asset, Question
 from country_workspace.contrib.kobo.raw.common import ListResponse
-from country_workspace.contrib.kobo.raw import asset as raw_asset, asset_list as raw_asset_list
+from country_workspace.contrib.kobo.raw import asset as raw_asset, asset_list as raw_asset_list, common as raw_common
 from country_workspace.contrib.kobo.raw import submission_list as raw_submission_list
 
 
-def handle_paginated_response[T, U](session: Session,
-                                    url: str,
-                                    collection_mapper: Callable[[ListResponse], list[T]],
-                                    item_mapper: Callable[[T], U]) -> Generator[U, None, None]:
+DataGetter = Callable[[str], Response]
+
+
+def _handle_paginated_response[T, U](data_getter: DataGetter,
+                                     url: str,
+                                     collection_mapper: Callable[[ListResponse], list[T]],
+                                     item_mapper: Callable[[T], U]) -> Generator[U, None, None]:
     while url:
-        response = session.get(url)
+        response = data_getter(url)
         response.raise_for_status()
         data: ListResponse = response.json()
         yield from map(item_mapper, collection_mapper(data))
         url = data["next"]
 
 
+def _get_raw_asset_list(data: raw_common.ListResponse) -> list[raw_asset_list.Asset]:
+    return [
+        datum for datum in
+        cast(raw_asset_list.AssetList, data)["results"]
+        if datum["asset_type"] == "survey"
+    ]
+
+
+def _get_raw_submission_list(data: raw_common.ListResponse) -> list[raw_submission_list.Submission]:
+    return cast(raw_submission_list.SubmissionList, data)["results"]
+
+
+def _get_asset_list(data_getter: DataGetter, url: str) -> Generator[Asset, None, None]:
+    return _handle_paginated_response(data_getter,
+                               url,
+                               _get_raw_asset_list,
+                               partial(_get_asset, data_getter))
+
+def _get_submission_list(data_getter: DataGetter, url: str, questions: list[Question]) -> Generator[Submission, None, None]:
+    return _handle_paginated_response(
+        data_getter,
+        url,
+        _get_raw_submission_list,
+        partial(Submission, data_getter, questions)
+    )
+
+
+def _get_asset(data_getter: DataGetter, raw: raw_asset_list.Asset) -> Asset:
+    response = data_getter(raw["url"])
+    response.raise_for_status()
+    data: raw_asset.Asset = response.json()
+    return Asset(data, partial(_get_submission_list, data_getter, raw["data"]))
+
+
+def _get_submission() -> Submission:
+    pass
+
+
 class Client:
     def __init__(self, *, base_url: str, token: str) -> None:
         self.base_url = base_url
-        self.session = Session()
-        self.session.auth = Auth(token)
+        session = Session()
+        session.auth = Auth(token)
+        self.data_getter: DataGetter = session.get
 
     @property
     def assets(self) -> Generator[Asset, None, None]:
-        yield from handle_paginated_response(self.session,
-                                             f"{self.base_url}/api/v2/assets.json",
-                                             lambda i: cast(raw_asset_list.AssetList, i)["results"],
-                                             self._get_asset_data)
-
-    def _get_asset_data(self, raw: raw_asset_list.Asset) -> Asset:
-        response = self.session.get(raw["url"])
-        response.raise_for_status()
-        data: raw_asset.Asset = response.json()
-        return Asset(data, self._get_asset_submissions(raw["data"]))
-
-    def _get_asset_submissions(self, url: str) -> Generator[Callable[[list[Question]], Submission], None, None]:
-        return handle_paginated_response(self.session,
-                                         url,
-                                         lambda i: cast(raw_submission_list.SubmissionList, i)["results"],
-                                         lambda i: partial(Submission, i))
+        yield from _get_asset_list(self.data_getter, f"{self.base_url}/api/v2/assets.json")
diff --git a/src/country_workspace/contrib/kobo/data.py b/src/country_workspace/contrib/kobo/data.py
index 0316e47..9f9d553 100644
--- a/src/country_workspace/contrib/kobo/data.py
+++ b/src/country_workspace/contrib/kobo/data.py
@@ -1,30 +1,94 @@
+from base64 import b64encode
 from collections import UserDict
 from collections.abc import Callable, Generator
+from enum import StrEnum, auto
+from functools import cached_property, reduce
+from requests import Response
+from typing import Any
 from uuid import UUID
 
 from country_workspace.contrib.kobo.raw import asset as raw_asset
 from country_workspace.contrib.kobo.raw import submission_list as raw_submission_list
 
 
+class SurveyItemType(StrEnum):
+    START = auto()
+    END = auto()
+    BEGIN_GROUP = auto()
+    END_GROUP = auto()
+    BEGIN_REPEAT = auto()
+    END_REPEAT = auto()
+
+
+
 class Raw[T]:
     def __init__(self, raw: T) -> None:
         self._raw = raw
 
 
-class Question(Raw[raw_asset.Question]):
+class Question(Raw[raw_asset.SurveyItem]):
+    def __init__(self, raw: raw_asset.SurveyItem, in_group: bool, in_roster: bool) -> None:
+        super().__init__(raw)
+        assert not (in_group and in_roster), "Cannot be both in group and roster"
+        self._in_group = in_group
+        self._in_roster = in_roster
+
+    def extract_answer(self, in_: raw_submission_list.Submission, out: dict[str, Any]) -> None:
+        if self._in_roster:
+            roster_key, _ = self.key.split("/")
+            roster = out.get(roster_key, [])
+            if self.key in in_:
+                if roster:
+                    roster[0][self.key] = in_.get(self.key)
+                else:
+                    roster.append({self.key: in_.get(self.key)})
+            else:
+                if roster_key in in_:
+                    for i, item in enumerate(in_[roster_key]):
+                        if len(roster) < i + 1:
+                            roster.append({})
+                        roster[i][self.key] = item[self.key]
+            out[roster_key] = roster
+        else:
+            out[self.key] = in_.get(self.key)
+
     @property
     def key(self) -> str:
-        return self._raw["$autoname"]
+        return self._raw["$xpath"]
 
     @property
     def labels(self) -> list[str]:
         return self._raw["label"]
 
 
+InAndOut = tuple[raw_submission_list.Submission, dict[str, Any]]
+
+
+def _extract_answer(in_and_out: InAndOut, question: Question) -> InAndOut:
+    question.extract_answer(*in_and_out)
+    return in_and_out
+
+
+def _download_attachments(data_getter: Callable[[str], Response], raw: raw_submission_list.Submission) -> None:
+    for attachment in raw["_attachments"]:
+        content = b64encode(data_getter(attachment["download_url"]).content).decode()
+        value = f"data:{attachment['mimetype']};base64,{content}"
+        key = attachment["question_xpath"]
+        if key in raw:
+            raw[key] = value
+        else:
+            parent, key = key.split("/")
+            parent, index = parent.split("[")
+            index = int(index.rstrip("]")) - 1
+            raw[parent][index][f"{parent}/{key}"] = value
+
+
 class Submission(Raw[raw_submission_list.Submission], UserDict):
-    def __init__(self, raw: raw_submission_list.Submission, questions: list[Question]) -> None:
+    def __init__(self, data_getter: Callable[[str], Response], questions: list[Question], raw: raw_submission_list.Submission) -> None:
         Raw.__init__(self, raw)
-        UserDict.__init__(self, {question.key: raw[question.key] for question in questions})
+        _download_attachments(data_getter, self._raw)
+        _, answers = reduce(_extract_answer, questions, (raw, {}))
+        UserDict.__init__(self, answers)
 
     @property
     def uuid(self) -> UUID:
@@ -32,7 +96,7 @@ def uuid(self) -> UUID:
 
 
 class Asset(Raw[raw_asset.Asset]):
-    def __init__(self, raw: raw_asset.Asset, submissions: Generator[Callable[[list[Question]], Submission], None, None]) -> None:
+    def __init__(self, raw: raw_asset.Asset, submissions: Callable[[list[Question]], Generator[Submission, None, None]]) -> None:
         super().__init__(raw)
         self._submissions = submissions
 
@@ -44,10 +108,27 @@ def uid(self) -> str:
     def name(self) -> str:
         return self._raw["name"]
 
-    @property
+    @cached_property
     def questions(self) -> list[Question]:
-        return [Question(raw_question) for raw_question in self._raw["content"]["survey"] if "label" in raw_question]
+        in_group = False
+        in_roster = False
+        questions = []
+        for raw_question in self._raw["content"]["survey"]:
+            match raw_question["type"]:
+                case SurveyItemType.START | SurveyItemType.END:
+                    pass
+                case SurveyItemType.BEGIN_GROUP:
+                    in_group = True
+                case SurveyItemType.END_GROUP:
+                    in_group = False
+                case SurveyItemType.BEGIN_REPEAT:
+                    in_roster = True
+                case SurveyItemType.END_REPEAT:
+                    in_roster = False
+                case _:
+                    questions.append(Question(raw_question, in_group, in_roster))
+        return questions
 
     @property
     def submissions(self) -> Generator[Submission, None, None]:
-        return (submission(self.questions) for submission in self._submissions)
+        yield from self._submissions(self.questions)
diff --git a/src/country_workspace/contrib/kobo/raw/asset.py b/src/country_workspace/contrib/kobo/raw/asset.py
index 4c5d9d7..67729a9 100644
--- a/src/country_workspace/contrib/kobo/raw/asset.py
+++ b/src/country_workspace/contrib/kobo/raw/asset.py
@@ -1,12 +1,11 @@
 from typing import TypedDict
 
-
-class Question(TypedDict("Question", {"$autoname": str})):
-    label: list[str]
+class SurveyItem(TypedDict("SurveyItem", {"$xpath": str})):
+    type: str
 
 
 class Content(TypedDict):
-    survey: list[Question]
+    survey: list[SurveyItem]
 
 
 class Asset(TypedDict):
diff --git a/src/country_workspace/contrib/kobo/raw/asset_list.py b/src/country_workspace/contrib/kobo/raw/asset_list.py
index 5393634..063debe 100644
--- a/src/country_workspace/contrib/kobo/raw/asset_list.py
+++ b/src/country_workspace/contrib/kobo/raw/asset_list.py
@@ -6,6 +6,7 @@
 class Asset(TypedDict):
     data: str
     url: str
+    asset_type: str
 
 
 class AssetList(ListResponse):
diff --git a/src/country_workspace/contrib/kobo/raw/submission_list.py b/src/country_workspace/contrib/kobo/raw/submission_list.py
index 9fddda1..951cc0b 100644
--- a/src/country_workspace/contrib/kobo/raw/submission_list.py
+++ b/src/country_workspace/contrib/kobo/raw/submission_list.py
@@ -3,8 +3,15 @@
 from country_workspace.contrib.kobo.raw.common import ListResponse
 
 
+class Attachment(TypedDict):
+    download_url: str
+    mimetype: str
+    question_xpath: str
+
+
 class Submission(TypedDict):
     _uuid: str
+    _attachments: list[Attachment]
 
 
 class SubmissionList(ListResponse):
diff --git a/tests/contrib/kobo/test_kobo_client.py b/tests/contrib/kobo/test_kobo_client.py
index 8a626d4..a3da54e 100644
--- a/tests/contrib/kobo/test_kobo_client.py
+++ b/tests/contrib/kobo/test_kobo_client.py
@@ -6,7 +6,7 @@
 from requests.sessions import Session
 from requests.exceptions import Timeout
 
-from country_workspace.contrib.kobo.client import handle_paginated_response
+from country_workspace.contrib.kobo.client import _handle_paginated_response
 
 
 SAMPLE_URL = "https://example.com"
@@ -32,7 +32,7 @@ def test_all_data_is_fetched() -> None:
     session.get.return_value.json.side_effect = responses
     assert (
         tuple(
-            handle_paginated_response(
+            _handle_paginated_response(
                 session, urls[0], lambda x: x["results"], identity
             )
         )
@@ -44,4 +44,4 @@ def test_error_is_propagated() -> None:
     session = Mock(spec=Session)
     session.get.return_value.raise_for_status.side_effect = Timeout
     with raises(Timeout):
-        tuple(handle_paginated_response(session, SAMPLE_URL, identity, identity))
+        tuple(_handle_paginated_response(session, SAMPLE_URL, identity, identity))

From 740d1427e93170d4a11e8766ecdba6b4ea895b60 Mon Sep 17 00:00:00 2001
From: Sergey Misuk <sergey.misuk@gmail.com>
Date: Thu, 2 Jan 2025 09:36:07 +0300
Subject: [PATCH 10/10] Refactor code

---
 src/country_workspace/admin/__init__.py       |  2 +-
 src/country_workspace/admin/job.py            |  6 +-
 .../config/fragments/constance.py             |  9 +-
 .../{admin/kobo.py => contrib/kobo/admin.py}  | 20 +---
 .../contrib/kobo/{raw => api}/__init__.py     |  0
 .../contrib/kobo/{ => api}/auth.py            |  4 +-
 .../contrib/kobo/{ => api}/client.py          | 12 +--
 .../contrib/kobo/{ => api}/data.py            | 20 ++--
 .../contrib/kobo/api/raw/__init__.py          |  0
 .../contrib/kobo/{ => api}/raw/asset.py       |  0
 .../contrib/kobo/{ => api}/raw/asset_list.py  |  2 +-
 .../contrib/kobo/{ => api}/raw/common.py      |  0
 .../kobo/{ => api}/raw/submission_list.py     |  2 +-
 src/country_workspace/contrib/kobo/forms.py   | 21 ++++
 src/country_workspace/contrib/kobo/models.py  | 12 +++
 src/country_workspace/contrib/kobo/sync.py    | 48 ++++++++++
 .../0002_alter_program_active_and_more.py     | 65 +++++++++++++
 ...02_koboasset_kobosyncjob_kobosubmission.py | 95 -------------------
 .../migrations/0003_koboasset_programs.py     | 18 ----
 .../migrations/0004_koboasset_name.py         | 18 ----
 .../migrations/0005_alter_koboasset_uid.py    | 18 ----
 ..._alter_kobosubmission_data_koboquestion.py | 31 ------
 ...0007_alter_koboquestion_unique_together.py | 17 ----
 src/country_workspace/models/__init__.py      |  4 +-
 src/country_workspace/models/jobs.py          |  8 --
 src/country_workspace/models/kobo.py          | 28 ------
 src/country_workspace/tasks.py                | 19 +---
 .../workspaces/admin/program.py               | 62 +++++++++++-
 .../program/_import_kobo_assets.html          | 14 +++
 .../workspace/program/_import_kobo_data.html  | 14 +++
 .../templates/workspace/program/import.html   | 21 +++-
 tests/contrib/kobo/test_auth.py               |  2 +-
 tests/contrib/kobo/test_kobo_client.py        |  2 +-
 33 files changed, 286 insertions(+), 308 deletions(-)
 rename src/country_workspace/{admin/kobo.py => contrib/kobo/admin.py} (50%)
 rename src/country_workspace/contrib/kobo/{raw => api}/__init__.py (100%)
 rename src/country_workspace/contrib/kobo/{ => api}/auth.py (84%)
 rename src/country_workspace/contrib/kobo/{ => api}/client.py (84%)
 rename src/country_workspace/contrib/kobo/{ => api}/data.py (89%)
 create mode 100644 src/country_workspace/contrib/kobo/api/raw/__init__.py
 rename src/country_workspace/contrib/kobo/{ => api}/raw/asset.py (100%)
 rename src/country_workspace/contrib/kobo/{ => api}/raw/asset_list.py (69%)
 rename src/country_workspace/contrib/kobo/{ => api}/raw/common.py (100%)
 rename src/country_workspace/contrib/kobo/{ => api}/raw/submission_list.py (79%)
 create mode 100644 src/country_workspace/contrib/kobo/forms.py
 create mode 100644 src/country_workspace/contrib/kobo/models.py
 create mode 100644 src/country_workspace/contrib/kobo/sync.py
 create mode 100644 src/country_workspace/migrations/0002_alter_program_active_and_more.py
 delete mode 100644 src/country_workspace/migrations/0002_koboasset_kobosyncjob_kobosubmission.py
 delete mode 100644 src/country_workspace/migrations/0003_koboasset_programs.py
 delete mode 100644 src/country_workspace/migrations/0004_koboasset_name.py
 delete mode 100644 src/country_workspace/migrations/0005_alter_koboasset_uid.py
 delete mode 100644 src/country_workspace/migrations/0006_alter_kobosubmission_data_koboquestion.py
 delete mode 100644 src/country_workspace/migrations/0007_alter_koboquestion_unique_together.py
 delete mode 100644 src/country_workspace/models/kobo.py
 create mode 100644 src/country_workspace/workspaces/templates/workspace/program/_import_kobo_assets.html
 create mode 100644 src/country_workspace/workspaces/templates/workspace/program/_import_kobo_data.html

diff --git a/src/country_workspace/admin/__init__.py b/src/country_workspace/admin/__init__.py
index 8176789..8cf37eb 100644
--- a/src/country_workspace/admin/__init__.py
+++ b/src/country_workspace/admin/__init__.py
@@ -16,7 +16,7 @@
 from .role import UserRoleAdmin  # noqa
 from .sync import SyncLog  # noqa
 from .user import UserAdmin  # noqa
-from .kobo import KoboAsset  # noqa
+from country_workspace.contrib.kobo.admin import KoboAsset  # noqa
 
 site.register(ContentType, admin_class=ContentTypeAdmin)
 site.register(Permission, admin_class=PermissionAdmin)
diff --git a/src/country_workspace/admin/job.py b/src/country_workspace/admin/job.py
index 1a14e4f..b9021c3 100644
--- a/src/country_workspace/admin/job.py
+++ b/src/country_workspace/admin/job.py
@@ -5,7 +5,7 @@
 from django.http import HttpRequest
 from django_celery_boost.admin import CeleryTaskModelAdmin
 
-from ..models import AsyncJob, KoboSyncJob
+from ..models import AsyncJob
 from .base import BaseModelAdmin
 from .filters import FailedFilter
 
@@ -29,7 +29,3 @@ def get_readonly_fields(self, request: "HttpRequest", obj: "AsyncJob | None" = N
         if obj:
             return "program", "batch", "owner", "local_status", "type", "action", "sentry_id"
         return super().get_readonly_fields(request, obj)
-
-@admin.register(KoboSyncJob)
-class KoboSyncJobAdmin(CeleryTaskModelAdmin, BaseModelAdmin):
-    pass
diff --git a/src/country_workspace/config/fragments/constance.py b/src/country_workspace/config/fragments/constance.py
index b102843..771f14d 100644
--- a/src/country_workspace/config/fragments/constance.py
+++ b/src/country_workspace/config/fragments/constance.py
@@ -1,4 +1,5 @@
 from .app import AURORA_API_TOKEN, AURORA_API_URL, HOPE_API_TOKEN, HOPE_API_URL, NEW_USER_DEFAULT_GROUP
+from .kobo import KOBO_TOKEN, KOBO_BASE_URL
 
 CONSTANCE_BACKEND = "constance.backends.database.DatabaseBackend"
 
@@ -46,8 +47,8 @@
     "AURORA_API_URL": (AURORA_API_URL, "Aurora API Server address", str),
     "HOPE_API_TOKEN": (HOPE_API_TOKEN, "HOPE API Access Token", "write_only_input"),
     "HOPE_API_URL": (HOPE_API_URL, "HOPE API Server address", str),
-    "KOBO_API_TOKEN": ("", "Kobo API Access Token", "write_only_input"),
-    "KOBO_API_URL": ("", "Kobo API Server address", str),
+    "KOBO_TOKEN": (KOBO_TOKEN, "Kobo API Access Token", "write_only_input"),
+    "KOBO_BASE_URL": (KOBO_BASE_URL, "Kobo Server address", str),
     "CACHE_TIMEOUT": (86400, "Cache Redis TTL", int),
     "CACHE_BY_VERSION": (False, "Invalidate Cache on CW version change", bool),
 }
@@ -60,7 +61,7 @@
         "AURORA_API_URL",
         "HOPE_API_TOKEN",
         "HOPE_API_URL",
-        "KOBO_API_TOKEN",
-        "KOBO_API_URL",
+        "KOBO_TOKEN",
+        "KOBO_BASE_URL",
     ),
 }
diff --git a/src/country_workspace/admin/kobo.py b/src/country_workspace/contrib/kobo/admin.py
similarity index 50%
rename from src/country_workspace/admin/kobo.py
rename to src/country_workspace/contrib/kobo/admin.py
index db9fdca..134c450 100644
--- a/src/country_workspace/admin/kobo.py
+++ b/src/country_workspace/contrib/kobo/admin.py
@@ -2,30 +2,14 @@
 
 from django.contrib import admin
 
-from .base import BaseModelAdmin
-from ..models import KoboAsset
-from ..models.kobo import KoboQuestion, KoboSubmission
-
-
-class ReadOnlyInlineAdmin(admin.TabularInline):
-    can_create = False
-    can_change = False
-    can_delete = False
-    extra = 0
-
-class KoboQuestionAdmin(ReadOnlyInlineAdmin):
-    model = KoboQuestion
-
-
-class KoboSubmissionAdmin(ReadOnlyInlineAdmin):
-    model = KoboSubmission
+from country_workspace.admin.base import BaseModelAdmin
+from country_workspace.models import KoboAsset
 
 
 @admin.register(KoboAsset)
 class KoboAssetAdmin(BaseModelAdmin):
     list_display = ("uid", "name")
     exclude = ("programs",)
-    inlines = (KoboQuestionAdmin, KoboSubmissionAdmin)
 
     def has_add_permission(self, *args: Any, **kwargs: Any) -> bool:
         return False
diff --git a/src/country_workspace/contrib/kobo/raw/__init__.py b/src/country_workspace/contrib/kobo/api/__init__.py
similarity index 100%
rename from src/country_workspace/contrib/kobo/raw/__init__.py
rename to src/country_workspace/contrib/kobo/api/__init__.py
diff --git a/src/country_workspace/contrib/kobo/auth.py b/src/country_workspace/contrib/kobo/api/auth.py
similarity index 84%
rename from src/country_workspace/contrib/kobo/auth.py
rename to src/country_workspace/contrib/kobo/api/auth.py
index aec0e44..7dd7df4 100644
--- a/src/country_workspace/contrib/kobo/auth.py
+++ b/src/country_workspace/contrib/kobo/api/auth.py
@@ -2,8 +2,8 @@
 from requests.models import PreparedRequest
 
 
-TOKEN = "Token"
-AUTHORIZATION = 'Authorization'
+TOKEN = "Token"  # noqa: S105
+AUTHORIZATION = "Authorization"
 
 class Auth(AuthBase):
     def __init__(self, api_key: str) -> None:
diff --git a/src/country_workspace/contrib/kobo/client.py b/src/country_workspace/contrib/kobo/api/client.py
similarity index 84%
rename from src/country_workspace/contrib/kobo/client.py
rename to src/country_workspace/contrib/kobo/api/client.py
index abcb32f..8f2e174 100644
--- a/src/country_workspace/contrib/kobo/client.py
+++ b/src/country_workspace/contrib/kobo/api/client.py
@@ -4,12 +4,12 @@
 from black.linegen import partial
 from requests import Session, Response
 
-from country_workspace.contrib.kobo.auth import Auth
-from country_workspace.contrib.kobo.data import Submission, Asset, Question
-from country_workspace.contrib.kobo.raw.common import ListResponse
-from country_workspace.contrib.kobo.raw import asset as raw_asset, asset_list as raw_asset_list, common as raw_common
-from country_workspace.contrib.kobo.raw import submission_list as raw_submission_list
-
+from country_workspace.contrib.kobo.api.auth import Auth
+from country_workspace.contrib.kobo.api.data import Submission, Asset, Question
+from country_workspace.contrib.kobo.api.raw.common import ListResponse
+from country_workspace.contrib.kobo.api.raw import asset_list as raw_asset_list
+from country_workspace.contrib.kobo.api.raw import asset as raw_asset, submission_list as raw_submission_list, \
+    common as raw_common
 
 DataGetter = Callable[[str], Response]
 
diff --git a/src/country_workspace/contrib/kobo/data.py b/src/country_workspace/contrib/kobo/api/data.py
similarity index 89%
rename from src/country_workspace/contrib/kobo/data.py
rename to src/country_workspace/contrib/kobo/api/data.py
index 9f9d553..29e4c03 100644
--- a/src/country_workspace/contrib/kobo/data.py
+++ b/src/country_workspace/contrib/kobo/api/data.py
@@ -7,8 +7,7 @@
 from typing import Any
 from uuid import UUID
 
-from country_workspace.contrib.kobo.raw import asset as raw_asset
-from country_workspace.contrib.kobo.raw import submission_list as raw_submission_list
+from country_workspace.contrib.kobo.api.raw import asset as raw_asset, submission_list as raw_submission_list
 
 
 class SurveyItemType(StrEnum):
@@ -42,12 +41,11 @@ def extract_answer(self, in_: raw_submission_list.Submission, out: dict[str, Any
                     roster[0][self.key] = in_.get(self.key)
                 else:
                     roster.append({self.key: in_.get(self.key)})
-            else:
-                if roster_key in in_:
-                    for i, item in enumerate(in_[roster_key]):
-                        if len(roster) < i + 1:
-                            roster.append({})
-                        roster[i][self.key] = item[self.key]
+            elif roster_key in in_:
+                for i, item in enumerate(in_[roster_key]):
+                    if len(roster) < i + 1:
+                        roster.append({})
+                    roster[i][self.key] = item[self.key]
             out[roster_key] = roster
         else:
             out[self.key] = in_.get(self.key)
@@ -60,6 +58,9 @@ def key(self) -> str:
     def labels(self) -> list[str]:
         return self._raw["label"]
 
+    def __str__(self) -> str:
+        return f"Question: {' '.join(self.labels)}"
+
 
 InAndOut = tuple[raw_submission_list.Submission, dict[str, Any]]
 
@@ -132,3 +133,6 @@ def questions(self) -> list[Question]:
     @property
     def submissions(self) -> Generator[Submission, None, None]:
         yield from self._submissions(self.questions)
+
+    def __str__(self) -> str:
+        return f"Asset: {self.name}"
diff --git a/src/country_workspace/contrib/kobo/api/raw/__init__.py b/src/country_workspace/contrib/kobo/api/raw/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/country_workspace/contrib/kobo/raw/asset.py b/src/country_workspace/contrib/kobo/api/raw/asset.py
similarity index 100%
rename from src/country_workspace/contrib/kobo/raw/asset.py
rename to src/country_workspace/contrib/kobo/api/raw/asset.py
diff --git a/src/country_workspace/contrib/kobo/raw/asset_list.py b/src/country_workspace/contrib/kobo/api/raw/asset_list.py
similarity index 69%
rename from src/country_workspace/contrib/kobo/raw/asset_list.py
rename to src/country_workspace/contrib/kobo/api/raw/asset_list.py
index 063debe..dda5427 100644
--- a/src/country_workspace/contrib/kobo/raw/asset_list.py
+++ b/src/country_workspace/contrib/kobo/api/raw/asset_list.py
@@ -1,6 +1,6 @@
 from typing import TypedDict
 
-from country_workspace.contrib.kobo.raw.common import ListResponse
+from country_workspace.contrib.kobo.api.raw.common import ListResponse
 
 
 class Asset(TypedDict):
diff --git a/src/country_workspace/contrib/kobo/raw/common.py b/src/country_workspace/contrib/kobo/api/raw/common.py
similarity index 100%
rename from src/country_workspace/contrib/kobo/raw/common.py
rename to src/country_workspace/contrib/kobo/api/raw/common.py
diff --git a/src/country_workspace/contrib/kobo/raw/submission_list.py b/src/country_workspace/contrib/kobo/api/raw/submission_list.py
similarity index 79%
rename from src/country_workspace/contrib/kobo/raw/submission_list.py
rename to src/country_workspace/contrib/kobo/api/raw/submission_list.py
index 951cc0b..3063392 100644
--- a/src/country_workspace/contrib/kobo/raw/submission_list.py
+++ b/src/country_workspace/contrib/kobo/api/raw/submission_list.py
@@ -1,6 +1,6 @@
 from typing import TypedDict
 
-from country_workspace.contrib.kobo.raw.common import ListResponse
+from country_workspace.contrib.kobo.api.raw.common import ListResponse
 
 
 class Attachment(TypedDict):
diff --git a/src/country_workspace/contrib/kobo/forms.py b/src/country_workspace/contrib/kobo/forms.py
new file mode 100644
index 0000000..c2ff29d
--- /dev/null
+++ b/src/country_workspace/contrib/kobo/forms.py
@@ -0,0 +1,21 @@
+from django import forms
+
+from country_workspace.contrib.kobo.models import KoboAsset
+
+
+class ImportKoboAssetsForm(forms.Form):
+    batch_name = forms.CharField(required=False, help_text="Label for this batch")
+
+
+class ImportKoboDataForm(forms.Form):
+    batch_name = forms.CharField(required=False, help_text="Label for this batch")
+    individual_records_field = forms.CharField(
+        required=False,
+        initial="individual_questions",
+        help_text="Which field contains individual records",
+    )
+    assets = forms.ModelMultipleChoiceField(
+        widget=forms.CheckboxSelectMultiple,
+        queryset=KoboAsset.objects.all(),
+        help_text="Which assets should be imported",
+    )
diff --git a/src/country_workspace/contrib/kobo/models.py b/src/country_workspace/contrib/kobo/models.py
new file mode 100644
index 0000000..667b9f6
--- /dev/null
+++ b/src/country_workspace/contrib/kobo/models.py
@@ -0,0 +1,12 @@
+from django.db import models
+
+from country_workspace.models import Program
+
+
+class KoboAsset(models.Model):
+    uid = models.CharField(primary_key=True, max_length=32, editable=False)
+    name = models.CharField(max_length=128, null=True, editable=False)
+    programs = models.ManyToManyField(Program)
+
+    def __str__(self) -> str:
+        return self.name or "No name"
diff --git a/src/country_workspace/contrib/kobo/sync.py b/src/country_workspace/contrib/kobo/sync.py
new file mode 100644
index 0000000..c5a465f
--- /dev/null
+++ b/src/country_workspace/contrib/kobo/sync.py
@@ -0,0 +1,48 @@
+from constance import config
+
+from country_workspace.contrib.kobo.api.client import Client as KoboClient
+from country_workspace.models import AsyncJob, KoboAsset, Batch, Individual
+from country_workspace.utils.fields import clean_field_name
+
+
+def sync_assets(_: AsyncJob) -> dict[str, int]:
+    client = KoboClient(base_url=config.KOBO_BASE_URL, token=config.KOBO_TOKEN)
+    created_assets = 0
+    updated_assets = 0
+    for asset in client.assets:
+        asset_model, created = KoboAsset.objects.update_or_create(uid=asset.uid, defaults={"name": asset.name})
+        if created:
+            created_assets += 1
+        else:
+            updated_assets += 1
+
+    return {"created": created_assets, "updated": updated_assets}
+
+
+def sync_data(job: AsyncJob) -> dict[str, int]:
+    batch = Batch.objects.create(
+        name=job.config["batch_name"],
+        program=job.program,
+        country_office=job.program.country_office,
+        imported_by=job.owner,
+        source=Batch.BatchSource.KOBO,
+    )
+    individual_records_field = job.config["individual_records_field"]
+    client = KoboClient(base_url=config.KOBO_BASE_URL, token=config.KOBO_TOKEN)
+    assets = (asset for asset in client.assets if asset.uid in job.config["assets"])
+    for asset in assets:
+        for submission in asset.submissions:
+            household_fields = {key: value for key, value in submission if key != individual_records_field}
+            household = batch.program.households.create(batch=batch, flex_fields={clean_field_name(key): value for key, value in household_fields.items()})
+            individuals = []
+            for individual in submission[individual_records_field]:
+                fullname = next((key for key in individual if key.startswith("given_name")), None)
+                individuals.append(
+                    Individual(
+                        batch=batch,
+                        household_id=household.pk,
+                        name=individual.get(fullname, ""),
+                        flex_fields={clean_field_name(key): value for key, value in individual.items()},
+                    ),
+                )
+    return {"households": 0, "individuals": 0}
diff --git a/src/country_workspace/migrations/0002_alter_program_active_and_more.py b/src/country_workspace/migrations/0002_alter_program_active_and_more.py
new file mode 100644
index 0000000..89d3929
--- /dev/null
+++ b/src/country_workspace/migrations/0002_alter_program_active_and_more.py
@@ -0,0 +1,65 @@
+# Generated by Django 5.1.3 on 2025-01-02 04:09
+
+import django.db.models.deletion
+import strategy_field.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("country_workspace", "0001_initial"),
+        ("hope_flex_fields", "0013_fielddefinition_validated_alter_datachecker_id_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="program",
+            name="active",
+            field=models.BooleanField(
+                default=False, help_text="Whether the program is active. Only active program are visible in the UI"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="program",
+            name="beneficiary_validator",
+            field=strategy_field.fields.StrategyField(
+                blank=True,
+                default="country_workspace.validators.registry.NoopValidator",
+                help_text="Validator to use to validate the whole Household",
+                null=True,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="program",
+            name="household_checker",
+            field=models.ForeignKey(
+                blank=True,
+                help_text="Checker to use with Household's records",
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="+",
+                to="hope_flex_fields.datachecker",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="program",
+            name="individual_checker",
+            field=models.ForeignKey(
+                blank=True,
+                help_text="Checker to use with Individual's records",
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="+",
+                to="hope_flex_fields.datachecker",
+            ),
+        ),
+        migrations.CreateModel(
+            name="KoboAsset",
+            fields=[
+                ("uid", models.CharField(editable=False, max_length=32, primary_key=True, serialize=False)),
+                ("name", models.CharField(editable=False, max_length=128, null=True)),
+                ("programs", models.ManyToManyField(to="country_workspace.program")),
+            ],
+        ),
+    ]
diff --git a/src/country_workspace/migrations/0002_koboasset_kobosyncjob_kobosubmission.py b/src/country_workspace/migrations/0002_koboasset_kobosyncjob_kobosubmission.py
deleted file mode 100644
index 49dbc10..0000000
--- a/src/country_workspace/migrations/0002_koboasset_kobosyncjob_kobosubmission.py
+++ /dev/null
@@ -1,95 +0,0 @@
-# Generated by Django 5.1.2 on 2024-11-21 09:02
-
-import concurrency.fields
-import django.db.models.deletion
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ("country_workspace", "0001_initial"),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name="KoboAsset",
-            fields=[
-                ("uid", models.CharField(max_length=32, primary_key=True, serialize=False)),
-            ],
-        ),
-        migrations.CreateModel(
-            name="KoboSyncJob",
-            fields=[
-                ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
-                ("version", concurrency.fields.AutoIncVersionField(default=0, help_text="record revision number")),
-                (
-                    "curr_async_result_id",
-                    models.CharField(
-                        blank=True,
-                        editable=False,
-                        help_text="Current (active) AsyncResult is",
-                        max_length=36,
-                        null=True,
-                    ),
-                ),
-                (
-                    "last_async_result_id",
-                    models.CharField(
-                        blank=True, editable=False, help_text="Latest executed AsyncResult is", max_length=36, null=True
-                    ),
-                ),
-                ("datetime_created", models.DateTimeField(auto_now_add=True, help_text="Creation date and time")),
-                (
-                    "datetime_queued",
-                    models.DateTimeField(
-                        blank=True, help_text="Queueing date and time", null=True, verbose_name="Queued At"
-                    ),
-                ),
-                (
-                    "repeatable",
-                    models.BooleanField(
-                        blank=True, default=False, help_text="Indicate if the job can be repeated as-is"
-                    ),
-                ),
-                ("celery_history", models.JSONField(blank=True, default=dict, editable=False)),
-                ("local_status", models.CharField(blank=True, default="", editable=False, max_length=100, null=True)),
-                (
-                    "group_key",
-                    models.CharField(
-                        blank=True,
-                        editable=False,
-                        help_text="Tasks with the same group key will not run in parallel",
-                        max_length=255,
-                        null=True,
-                    ),
-                ),
-                (
-                    "owner",
-                    models.ForeignKey(
-                        blank=True,
-                        null=True,
-                        on_delete=django.db.models.deletion.CASCADE,
-                        related_name="%(app_label)s_%(class)s_jobs",
-                        to=settings.AUTH_USER_MODEL,
-                    ),
-                ),
-            ],
-            options={
-                "abstract": False,
-                "default_permissions": ("add", "change", "delete", "view", "queue", "terminate", "inspect", "revoke"),
-            },
-        ),
-        migrations.CreateModel(
-            name="KoboSubmission",
-            fields=[
-                ("uuid", models.UUIDField(primary_key=True, serialize=False)),
-                ("data", models.JSONField()),
-                (
-                    "asset",
-                    models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="country_workspace.koboasset"),
-                ),
-            ],
-        ),
-    ]
diff --git a/src/country_workspace/migrations/0003_koboasset_programs.py b/src/country_workspace/migrations/0003_koboasset_programs.py
deleted file mode 100644
index 54836fb..0000000
--- a/src/country_workspace/migrations/0003_koboasset_programs.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 5.1.2 on 2024-11-25 09:25
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ("country_workspace", "0002_koboasset_kobosyncjob_kobosubmission"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="koboasset",
-            name="programs",
-            field=models.ManyToManyField(to="country_workspace.program"),
-        ),
-    ]
diff --git a/src/country_workspace/migrations/0004_koboasset_name.py b/src/country_workspace/migrations/0004_koboasset_name.py
deleted file mode 100644
index cc3f604..0000000
--- a/src/country_workspace/migrations/0004_koboasset_name.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 5.1.2 on 2024-11-25 20:14
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ("country_workspace", "0003_koboasset_programs"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="koboasset",
-            name="name",
-            field=models.CharField(editable=False, max_length=128, null=True),
-        ),
-    ]
diff --git a/src/country_workspace/migrations/0005_alter_koboasset_uid.py b/src/country_workspace/migrations/0005_alter_koboasset_uid.py
deleted file mode 100644
index ef8844a..0000000
--- a/src/country_workspace/migrations/0005_alter_koboasset_uid.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Generated by Django 5.1.2 on 2024-11-25 20:19
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ("country_workspace", "0004_koboasset_name"),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name="koboasset",
-            name="uid",
-            field=models.CharField(editable=False, max_length=32, primary_key=True, serialize=False),
-        ),
-    ]
diff --git a/src/country_workspace/migrations/0006_alter_kobosubmission_data_koboquestion.py b/src/country_workspace/migrations/0006_alter_kobosubmission_data_koboquestion.py
deleted file mode 100644
index 175ac4e..0000000
--- a/src/country_workspace/migrations/0006_alter_kobosubmission_data_koboquestion.py
+++ /dev/null
@@ -1,31 +0,0 @@
-# Generated by Django 5.1.2 on 2024-11-25 20:41
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ("country_workspace", "0005_alter_koboasset_uid"),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name="kobosubmission",
-            name="data",
-            field=models.JSONField(default=dict),
-        ),
-        migrations.CreateModel(
-            name="KoboQuestion",
-            fields=[
-                ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
-                ("key", models.CharField(editable=False, max_length=128, null=True)),
-                ("labels", models.JSONField(default=list)),
-                (
-                    "asset",
-                    models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="country_workspace.koboasset"),
-                ),
-            ],
-        ),
-    ]
diff --git a/src/country_workspace/migrations/0007_alter_koboquestion_unique_together.py b/src/country_workspace/migrations/0007_alter_koboquestion_unique_together.py
deleted file mode 100644
index 241acc1..0000000
--- a/src/country_workspace/migrations/0007_alter_koboquestion_unique_together.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Generated by Django 5.1.2 on 2024-11-25 21:54
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ("country_workspace", "0006_alter_kobosubmission_data_koboquestion"),
-    ]
-
-    operations = [
-        migrations.AlterUniqueTogether(
-            name="koboquestion",
-            unique_together={("asset", "key")},
-        ),
-    ]
diff --git a/src/country_workspace/models/__init__.py b/src/country_workspace/models/__init__.py
index 2e01086..475deef 100644
--- a/src/country_workspace/models/__init__.py
+++ b/src/country_workspace/models/__init__.py
@@ -1,7 +1,7 @@
 from .batch import Batch  # noqa
 from .household import Household  # noqa
 from .individual import Individual  # noqa
-from .jobs import AsyncJob, KoboSyncJob  # noqa
+from .jobs import AsyncJob  # noqa
 from .locations import Area, AreaType, Country  # noqa
 from .office import Office  # noqa
 from .program import Program  # noqa
@@ -9,4 +9,4 @@
 from .role import UserRole  # noqa
 from .sync import SyncLog  # noqa
 from .user import User  # noqa
-from .kobo import KoboAsset, KoboSubmission, KoboQuestion  # noqa
+from country_workspace.contrib.kobo.models import KoboAsset  # noqa
diff --git a/src/country_workspace/models/jobs.py b/src/country_workspace/models/jobs.py
index e3c031b..03d7cf0 100644
--- a/src/country_workspace/models/jobs.py
+++ b/src/country_workspace/models/jobs.py
@@ -1,7 +1,6 @@
 from typing import Any, Callable
 
 from django.apps import apps
-from django.core.exceptions import ValidationError
 from django.db import models
 from django.utils.module_loading import import_string
 
@@ -62,10 +61,3 @@ def execute(self) -> Any:
             if sid:
                 self.sentry_id = sid
                 self.save(update_fields=["sentry_id"])
-
-class KoboSyncJob(CeleryTaskModel):
-    celery_task_name = "country_workspace.tasks.sync_kobo_assets_task"
-
-    def clean(self) -> None:
-        if self.__class__.objects.exists() and not self.pk:
-            raise ValidationError(f"You can have only one {self.__class__.__name__} instance.")
diff --git a/src/country_workspace/models/kobo.py b/src/country_workspace/models/kobo.py
deleted file mode 100644
index fe291c5..0000000
--- a/src/country_workspace/models/kobo.py
+++ /dev/null
@@ -1,28 +0,0 @@
-from django.db import models
-
-from country_workspace.models import Program
-
-
-class KoboAsset(models.Model):
-    uid = models.CharField(primary_key=True, max_length=32, editable=False)
-    name = models.CharField(max_length=128, null=True, editable=False)
-    programs = models.ManyToManyField(Program)
-
-    def __str__(self) -> str:
-        return self.name or "No name"
-
-
-
-class KoboQuestion(models.Model):
-    asset = models.ForeignKey(KoboAsset, on_delete=models.CASCADE)
-    key = models.CharField(max_length=128, null=True, editable=False)
-    labels = models.JSONField(default=list)
-
-    class Meta:
-        unique_together = ("asset", "key")
-
-
-class KoboSubmission(models.Model):
-    uuid = models.UUIDField(primary_key=True)
-    asset = models.ForeignKey(KoboAsset, on_delete=models.CASCADE)
-    data = models.JSONField(default=dict)
diff --git a/src/country_workspace/tasks.py b/src/country_workspace/tasks.py
index 6c91cfe..abb918c 100644
--- a/src/country_workspace/tasks.py
+++ b/src/country_workspace/tasks.py
@@ -2,16 +2,12 @@
 import logging
 from typing import TYPE_CHECKING, Any
 
-from constance import config
 from django.core.cache import cache
 import sentry_sdk
 from redis_lock import Lock
 
 from country_workspace.config.celery import app
-from country_workspace.contrib.kobo.client import Client as KoboClient
-from country_workspace.models import AsyncJob, KoboAsset, KoboSubmission
-from country_workspace.models.jobs import KoboSyncJob
-from country_workspace.models.kobo import KoboQuestion
+from country_workspace.models import AsyncJob
 
 logger = logging.getLogger(__name__)
 
@@ -64,16 +60,3 @@ def sync_job_task(pk: int, version: int) -> dict[str, Any]:
 @app.task()
 def removed_expired_jobs(**kwargs: Any) -> None:
     AsyncJob.objects.filter(**kwargs).delete()
-
-@app.task
-def sync_kobo_assets_task(job_id: int, version: int) -> None:
-    _ = KoboSyncJob.objects.get(pk=job_id, version=version)
-    client = KoboClient(base_url=config.KOBO_BASE_URL, token=config.KOBO_TOKEN)
-    for asset_data in client.assets:
-        asset_model, _ = KoboAsset.objects.update_or_create(uid=asset_data.uid, defaults={"name": asset_data.name})
-
-        for question_data in asset_data.questions:
-            KoboQuestion.objects.update_or_create(asset=asset_model, key=question_data.key, labels=question_data.labels)
-
-        for submission_data in asset_data.submissions:
-            KoboSubmission.objects.update_or_create(uuid=submission_data.uuid, asset=asset_model, data=submission_data.data)
diff --git a/src/country_workspace/workspaces/admin/program.py b/src/country_workspace/workspaces/admin/program.py
index 2df8261..f562d77 100644
--- a/src/country_workspace/workspaces/admin/program.py
+++ b/src/country_workspace/workspaces/admin/program.py
@@ -19,6 +19,8 @@
 from country_workspace.state import state
 
 from ...contrib.aurora.forms import ImportAuroraForm
+from ...contrib.kobo.forms import ImportKoboDataForm, ImportKoboAssetsForm
+from ...contrib.kobo.sync import sync_assets as sync_kobo_assets, sync_data as sync_kobo_data
 from ...datasources.rdi import import_from_rdi
 from ...models import AsyncJob
 from ...utils.flex_fields import get_checker_fields
@@ -226,6 +228,8 @@ def import_data(self, request: HttpRequest, pk: str) -> "HttpResponse":
         context["media"] = Media(js=["admin/js/vendor/jquery/jquery.js", "workspace/js/import_data.js"], css={})
         form_rdi = ImportFileForm(prefix="rdi")
         form_aurora = ImportAuroraForm(prefix="aurora")
+        form_kobo_assets = ImportKoboAssetsForm(prefix="kobo_assets")
+        form_kobo_data = ImportKoboDataForm(prefix="kobo_data")
 
         if request.method == "POST":
             match request.POST.get("_selected_tab"):
@@ -235,11 +239,17 @@ def import_data(self, request: HttpRequest, pk: str) -> "HttpResponse":
                 case "aurora":
                     if not (form_aurora := self.import_aurora(request, program)):
                         return HttpResponseRedirect(reverse("workspace:workspaces_countryasyncjob_changelist"))
-                case "kobo":
-                    self.message_user(request, _("Not implemented"))
+                case "kobo-assets":
+                    if not (form_kobo_data := self.import_kobo_assets(request, program)):
+                        return HttpResponseRedirect(reverse("workspace:workspaces_countryasyncjob_changelist"))
+                case "kobo-data":
+                    if not (form_kobo_data := self.import_kobo_data(request, program)):
+                        return HttpResponseRedirect(reverse("workspace:workspaces_countryasyncjob_changelist"))
 
         context["form_rdi"] = form_rdi
         context["form_aurora"] = form_aurora
+        context["form_kobo_assets"] = form_kobo_assets
+        context["form_kobo_data"] = form_kobo_data
 
         return render(request, "workspace/program/import.html", context)
 
@@ -287,3 +297,51 @@ def import_aurora(self, request: HttpRequest, program: "CountryProgram") -> "Imp
             )
             return None
         return form
+
+    def import_kobo_assets(self, request: HttpRequest, program: "CountryProgram") -> ImportKoboAssetsForm | None:
+        form = ImportKoboAssetsForm(request.POST, prefix="kobo_assets")
+        if form.is_valid():
+            job: AsyncJob = AsyncJob.objects.create(
+                type=AsyncJob.JobType.TASK,
+                action=fqn(sync_kobo_assets),
+                file=None,
+                program=program,
+                owner=request.user,
+                config={
+                    "batch_name": form.cleaned_data["batch_name"] or BATCH_NAME_DEFAULT,
+                }
+            )
+            job.queue()
+            self.message_user(
+                request,
+                _("The Kobo assets import task has been successfully queued. Job #{0}.").format(job.id),
+                level=messages.SUCCESS
+            )
+            return None
+
+        return form
+
+    def import_kobo_data(self, request: HttpRequest, program: "CountryProgram") -> ImportKoboDataForm | None:
+        form = ImportKoboDataForm(request.POST, prefix="kobo_data")
+        if form.is_valid():
+            job: AsyncJob = AsyncJob.objects.create(
+                type=AsyncJob.JobType.TASK,
+                action=fqn(sync_kobo_data),
+                file=None,
+                program=program,
+                owner=request.user,
+                config={
+                    "batch_name": form.cleaned_data["batch_name"] or BATCH_NAME_DEFAULT,
+                    "assets": list(form.cleaned_data["assets"].values_list("uid", flat=True)),
+                    "individual_records_field": form.cleaned_data["individual_records_field"],
+                }
+            )
+            job.queue()
+            self.message_user(
+                request,
+                _("The Kobo data import task has been successfully queued. Job #{0}.").format(job.id),
+                level=messages.SUCCESS
+            )
+            return None
+
+        return form
diff --git a/src/country_workspace/workspaces/templates/workspace/program/_import_kobo_assets.html b/src/country_workspace/workspaces/templates/workspace/program/_import_kobo_assets.html
new file mode 100644
index 0000000..eb073d8
--- /dev/null
+++ b/src/country_workspace/workspaces/templates/workspace/program/_import_kobo_assets.html
@@ -0,0 +1,14 @@
+{% load i18n workspace_modify workspace_urls %}
+<form method="post" id="import-kobo" enctype="multipart/form-data">
+    <input type="hidden" name="_selected_tab" value="kobo_assets">
+    {% csrf_token %}
+    <table>
+        {{ form_kobo_assets.as_table }}
+    </table>
+    <div class="submit-row">
+        <input type="submit" value="{% translate 'Import' %}" class="default" name="_save">
+        {% url opts|workspace_urlname:'change' original.pk as changelist_url %}
+        <a href="{% add_preserved_filters changelist_url %}"
+           class="closelink">{% translate 'Close' %}</a>
+    </div>
+</form>
diff --git a/src/country_workspace/workspaces/templates/workspace/program/_import_kobo_data.html b/src/country_workspace/workspaces/templates/workspace/program/_import_kobo_data.html
new file mode 100644
index 0000000..d87b55b
--- /dev/null
+++ b/src/country_workspace/workspaces/templates/workspace/program/_import_kobo_data.html
@@ -0,0 +1,14 @@
+{% load i18n workspace_modify workspace_urls %}
+<form method="post" id="import-kobo" enctype="multipart/form-data">
+    <input type="hidden" name="_selected_tab" value="kobo_data">
+    {% csrf_token %}
+    <table>
+        {{ form_kobo_data.as_table }}
+    </table>
+    <div class="submit-row">
+        <input type="submit" value="{% translate 'Import' %}" class="default" name="_save">
+        {% url opts|workspace_urlname:'change' original.pk as changelist_url %}
+        <a href="{% add_preserved_filters changelist_url %}"
+           class="closelink">{% translate 'Close' %}</a>
+    </div>
+</form>
diff --git a/src/country_workspace/workspaces/templates/workspace/program/import.html b/src/country_workspace/workspaces/templates/workspace/program/import.html
index 0a3120e..66223d4 100644
--- a/src/country_workspace/workspaces/templates/workspace/program/import.html
+++ b/src/country_workspace/workspaces/templates/workspace/program/import.html
@@ -32,10 +32,18 @@
                         </li>
                         <li class="me-2" role="presentation">
                             <button
-                                    data-tabs-target="#tab-kobo"
-                                    data-input-value="kobo"
+                                    data-tabs-target="#tab-kobo-assets"
+                                    data-input-value="kobo-assets"
                                     type="button" role="tab"
-                                    aria-selected="false">Kobo
+                                    aria-selected="false">Kobo Assets
+                            </button>
+                        </li>
+                        <li class="me-2" role="presentation">
+                            <button
+                                    data-tabs-target="#tab-kobo-data"
+                                    data-input-value="kobo-data"
+                                    type="button" role="tab"
+                                    aria-selected="false">Kobo Data
                             </button>
                         </li>
                     </ul>
@@ -48,8 +56,11 @@
                     <div class="hidden content" id="tab-aurora">
                         {% include "workspace/program/_import_aurora.html" %}
                     </div>
-                    <div class="hidden content" id="tab-kobo">
-                        Kobo
+                    <div class="hidden content" id="tab-kobo-assets">
+                        {% include "workspace/program/_import_kobo_assets.html" %}
+                    </div>
+                    <div class="hidden content" id="tab-kobo-data">
+                        {% include "workspace/program/_import_kobo_data.html" %}
                     </div>
                 </div>
             </div>
diff --git a/tests/contrib/kobo/test_auth.py b/tests/contrib/kobo/test_auth.py
index 3060452..9ef1857 100644
--- a/tests/contrib/kobo/test_auth.py
+++ b/tests/contrib/kobo/test_auth.py
@@ -2,7 +2,7 @@
 
 from requests.models import PreparedRequest
 
-from country_workspace.contrib.kobo.auth import Auth, AUTHORIZATION, TOKEN
+from country_workspace.contrib.kobo.api.auth import Auth, AUTHORIZATION, TOKEN
 
 
 def test_token_is_used() -> None:
diff --git a/tests/contrib/kobo/test_kobo_client.py b/tests/contrib/kobo/test_kobo_client.py
index a3da54e..62087ab 100644
--- a/tests/contrib/kobo/test_kobo_client.py
+++ b/tests/contrib/kobo/test_kobo_client.py
@@ -6,7 +6,7 @@
 from requests.sessions import Session
 from requests.exceptions import Timeout
 
-from country_workspace.contrib.kobo.client import _handle_paginated_response
+from country_workspace.contrib.kobo.api.client import _handle_paginated_response
 
 
 SAMPLE_URL = "https://example.com"