add ! hde client

pyproject.toml

@@ -51,6 +51,7 @@ dependencies = [
     "sentry-sdk>=2.7.1",
     "social-auth-app-django",
     "unicef-security>=1.5.1",
+    "pydantic>=2.10.1",
 ]
 [project.scripts]
 celery-monitor = "country_workspace.__monitor__:run"

src/country_workspace/config/__init__.py

@@ -100,6 +100,8 @@ class Group(Enum):
         "https://django-environ.readthedocs.io/en/latest/types.html#environ-env-db-url",
     ),
     "DEBUG": (bool, False, True, False, setting("debug")),
+    "DEDUP_ENGINE_API_TOKEN": (str, "", "", False, "Dedup engine API token"),
+    "DEDUP_ENGINE_API_URL": (str, "", "", False, "Dedup engine API url"),
     # "EMAIL_BACKEND": (
     #     str,
     #     "django.core.mail.backends.smtp.EmailBackend",

src/country_workspace/config/fragments/app.py

@@ -12,6 +12,10 @@
 HOPE_API_TOKEN = env("HOPE_API_TOKEN")
 HOPE_API_URL = env("HOPE_API_URL")
 
+DEDUP_ENGINE_API_TOKEN = env("DEDUP_ENGINE_API_TOKEN")
+DEDUP_ENGINE_API_URL = env("DEDUP_ENGINE_API_URL")
+
+
 HH_LOOKUPS = [
     "ResidenceStatus",
 ]

src/country_workspace/config/fragments/constance.py

@@ -1,4 +1,12 @@
-from .app import AURORA_API_TOKEN, AURORA_API_URL, HOPE_API_TOKEN, HOPE_API_URL, NEW_USER_DEFAULT_GROUP
+from country_workspace.config.fragments.app import (
+    AURORA_API_TOKEN,
+    AURORA_API_URL,
+    DEDUP_ENGINE_API_TOKEN,
+    DEDUP_ENGINE_API_URL,
+    HOPE_API_TOKEN,
+    HOPE_API_URL,
+    NEW_USER_DEFAULT_GROUP,
+)
 
 CONSTANCE_BACKEND = "constance.backends.database.DatabaseBackend"
 
@@ -46,6 +54,9 @@
     "AURORA_API_URL": (AURORA_API_URL, "Aurora API Server address", str),
     "HOPE_API_TOKEN": (HOPE_API_TOKEN, "HOPE API Access Token", "write_only_input"),
     "HOPE_API_URL": (HOPE_API_URL, "HOPE API Server address", str),
+    "DEDUP_ENGINE_API_TOKEN": (DEDUP_ENGINE_API_TOKEN, "Dedup engine API Access Token", "write_only_input"),
+    "DEDUP_ENGINE_API_URL": (DEDUP_ENGINE_API_URL, "Dedup engine API Server address", str),
+    "KOBO_API_URL": ("", "Kobo API Server address", str),
     "KOBO_API_TOKEN": ("", "Kobo API Access Token", "write_only_input"),
     "KOBO_API_URL": ("", "Kobo API Server address", str),
     "CACHE_TIMEOUT": (86400, "Cache Redis TTL", int),
@@ -58,6 +69,8 @@
     "Remote System Tokens": (
         "AURORA_API_TOKEN",
         "AURORA_API_URL",
+        "DEDUP_ENGINE_API_TOKEN",
+        "DEDUP_ENGINE_API_URL",
         "HOPE_API_TOKEN",
         "HOPE_API_URL",
         "KOBO_API_TOKEN",

src/country_workspace/config/settings.py

@@ -50,6 +50,7 @@
     "country_workspace.security",
     "country_workspace.apps.HCWConfig",
     "country_workspace.workspaces.apps.Config",
+    "country_workspace.contrib.dedup_engine.apps.Config",
     "country_workspace.versioning",
     "country_workspace.cache",
     # these should be optional in the future

src/country_workspace/contrib/dedup_engine/adapters/__init__.py

@@ -0,0 +1,4 @@
+from .deduplication_set import DeduplicationSetAdapter  # noqa
+from .duplicate import DuplicateAdapter  # noqa
+from .ignored import IgnoredAdapter  # noqa
+from .image import ImageAdapter  # noqa

src/country_workspace/contrib/dedup_engine/adapters/base.py

@@ -0,0 +1,72 @@
+from enum import Enum
+from typing import Any, Generic, Type
+
+from requests import Session
+
+from country_workspace.contrib.dedup_engine.adapters.mixins import URLMixin, ValidationMixin
+from country_workspace.contrib.dedup_engine.endpoints import Endpoint
+from country_workspace.contrib.dedup_engine.types import TCreate, TModel
+
+
+class HTTPMethod(Enum):
+    GET = "GET"
+    POST = "POST"
+    DELETE = "DELETE"
+
+
+class BaseAdapter(Generic[TModel, TCreate], URLMixin, ValidationMixin):
+    def __init__(
+        self, session: Session, endpoints: Endpoint, model_class: Type[TModel], create_class: Type[TCreate] = None
+    ) -> None:
+        self.session = session
+        self.endpoints = endpoints
+        self.model_class = model_class
+        self.create_class = create_class or model_class
+
+    def list(self, url_path: str, **kwargs) -> list[TModel]:
+        url = self.prepare_url(url_path, **kwargs)
+        response = self._request(HTTPMethod.GET.value, url)
+        return [self.model_class(**item) for item in response.json()]
+
+    def retrieve(self, url_path: str, **kwargs) -> TModel:
+        url = self.prepare_url(url_path, **kwargs)
+        response = self._request(HTTPMethod.GET.value, url)
+        return self.model_class(**response.json())
+
+    def create(self, url_path: str, data: TCreate, **kwargs) -> TModel:
+        url = self.prepare_url(url_path, **kwargs)
+        response = self._request(
+            HTTPMethod.POST.value, url, json=self.validate_data(data, self.create_class).model_dump(by_alias=True)
+        )
+        print(f"{response.json()=}")
+        return self.model_class(**response.json())
+
+    def destroy(self, url_path: str, **kwargs) -> None:
+        url = self.prepare_url(url_path, **kwargs)
+        response = self._request(HTTPMethod.DELETE.value, url)
+        if response.status_code != 204:
+            response.raise_for_status()
+
+    def update(self, url_path: str, data: TModel, **kwargs) -> TModel:
+        raise NotImplementedError("Update method is not implemented")
+
+    def _request(self, method: str, url: str, **kwargs) -> Any:
+        print(f"{method} {url=}")
+        response = self.session.request(method, url, **kwargs)
+        response.raise_for_status()
+        return response
+
+    def _action(
+        self,
+        method: HTTPMethod,
+        url_path: str,
+        *,
+        path_params: dict[str, Any] = None,
+        data: Any = None,
+    ) -> Any:
+        path_params = path_params or {}
+        url = self.prepare_url(url_path, **path_params)
+        response = self._request(method.value, url, json=data)
+        if response.content:
+            return response.json()
+        return None

src/country_workspace/contrib/dedup_engine/adapters/deduplication_set.py

@@ -0,0 +1,39 @@
+from typing import override
+from uuid import UUID
+
+from country_workspace.contrib.dedup_engine.adapters.base import BaseAdapter, HTTPMethod
+from country_workspace.contrib.dedup_engine.models import DeduplicationSet, DeduplicationSetCreate
+
+
+class DeduplicationSetAdapter(BaseAdapter[DeduplicationSet, DeduplicationSetCreate]):
+    @override
+    def list(self) -> list[DeduplicationSet]:
+        return super().list(url_path=self.endpoints.deduplication_set)
+
+    @override
+    def retrieve(self, data: DeduplicationSet | UUID) -> DeduplicationSet:
+        return super().retrieve(
+            url_path=self.endpoints.deduplication_set_detail,
+            deduplication_set_id=self.get_entity_id(data),
+        )
+
+    @override
+    def create(self, data: DeduplicationSetCreate) -> DeduplicationSet:
+        return super().create(
+            url_path=self.endpoints.deduplication_set,
+            data=data,
+        )
+
+    @override
+    def destroy(self, data: DeduplicationSet | UUID) -> None:
+        super().destroy(
+            url_path=self.endpoints.deduplication_set_detail,
+            deduplication_set_id=self.get_entity_id(data),
+        )
+
+    def process(self, data: DeduplicationSet | UUID) -> None:
+        self._action(
+            HTTPMethod.POST,
+            self.endpoints.process,
+            path_params={"deduplication_set_id": self.get_entity_id(data)},
+        )

src/country_workspace/contrib/dedup_engine/adapters/duplicate.py

@@ -0,0 +1,26 @@
+from typing import override
+from uuid import UUID
+
+from country_workspace.contrib.dedup_engine.adapters.base import BaseAdapter
+from country_workspace.contrib.dedup_engine.models import DeduplicationSet, Duplicate
+
+
+class DuplicateAdapter(BaseAdapter[Duplicate, None]):
+    @override
+    def list(self, deduplication_set: DeduplicationSet | UUID) -> list[Duplicate]:
+        return super().list(
+            url_path=self.endpoints.duplicate,
+            deduplication_set_id=self.get_entity_id(deduplication_set),
+        )
+
+    @override
+    def retrieve(self, *args, **kwargs) -> None:
+        raise NotImplementedError("Retrieval of Duplicate objects is not supported.")
+
+    @override
+    def create(self, *args, **kwargs) -> None:
+        raise NotImplementedError("Creation of Duplicate objects is not supported.")
+
+    @override
+    def destroy(self, *args, **kwargs) -> None:
+        raise NotImplementedError("Deletion of Duplicate objects is not supported.")

src/country_workspace/contrib/dedup_engine/adapters/ignored.py

@@ -0,0 +1,45 @@
+from typing import Type, override
+from uuid import UUID
+
+from requests import Session
+
+from country_workspace.contrib.dedup_engine.adapters.base import BaseAdapter, Endpoint
+from country_workspace.contrib.dedup_engine.models import DeduplicationSet, Ignored, IgnoredCreate
+
+
+class IgnoredAdapter(BaseAdapter[Ignored, IgnoredCreate]):
+    def __init__(
+        self,
+        session: Session,
+        endpoints: Endpoint,
+        model_class: Type[Ignored],
+        create_class: Type[IgnoredCreate],
+        resource_type: str,
+    ) -> None:
+        super().__init__(session, endpoints, model_class, create_class)
+        self.resource_type = resource_type
+
+    @override
+    def list(self, deduplication_set: DeduplicationSet | UUID) -> list[Ignored]:
+        return super().list(
+            url_path=self.endpoints.ignored,
+            deduplication_set_id=self.get_entity_id(deduplication_set),
+            resource_type=self.resource_type,
+        )
+
+    @override
+    def create(self, deduplication_set: DeduplicationSet | UUID, data: IgnoredCreate) -> Ignored:
+        return super().create(
+            url_path=self.endpoints.ignored,
+            deduplication_set_id=self.get_entity_id(deduplication_set),
+            resource_type=self.resource_type,
+            data=data,
+        )
+
+    @override
+    def retrieve(self, *args, **kwargs) -> None:
+        raise NotImplementedError("Retrieval of Ignored objects is not supported.")
+
+    @override
+    def destroy(self, *args, **kwargs) -> None:
+        raise NotImplementedError("Deletion of Ignored objects is not supported.")

src/country_workspace/contrib/dedup_engine/adapters/image.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+from typing import override
+from uuid import UUID
+
+from country_workspace.contrib.dedup_engine.adapters.base import BaseAdapter, HTTPMethod
+from country_workspace.contrib.dedup_engine.models import DeduplicationSet, Image, ImageCreate
+
+
+class ImageAdapter(BaseAdapter[Image, ImageCreate]):
+
+    @override
+    def list(self, deduplication_set: DeduplicationSet | UUID) -> list[Image]:
+        return super().list(
+            url_path=self.endpoints.image,
+            deduplication_set_id=self.get_entity_id(deduplication_set),
+        )
+
+    @override
+    def create(self, deduplication_set: DeduplicationSet | UUID, data: ImageCreate) -> Image:
+        return super().create(
+            url_path=self.endpoints.image,
+            deduplication_set_id=self.get_entity_id(deduplication_set),
+            data=data,
+        )
+
+    @override
+    def destroy(self, deduplication_set: DeduplicationSet | UUID, image: Image | UUID) -> None:
+        return super().destroy(
+            url_path=self.endpoints.image_detail,
+            deduplication_set_id=self.get_entity_id(deduplication_set),
+            image_id=self.get_entity_id(image),
+        )
+
+    def create_bulk(self, deduplication_set: DeduplicationSet | UUID, data: list[ImageCreate]) -> list[Image]:
+        validated_data = [self.validate_data(item, ImageCreate).model_dump(by_alias=True) for item in data]
+        response_data = self._action(
+            HTTPMethod.POST,
+            self.endpoints.image_bulk,
+            path_params={"deduplication_set_id": self.get_entity_id(deduplication_set)},
+            data=validated_data,
+        )
+        return [Image(**item) for item in response_data]
+
+    def destroy_bulk(self, deduplication_set: DeduplicationSet | UUID) -> None:
+        self._action(
+            HTTPMethod.DELETE,
+            self.endpoints.image_bulk_clear,
+            path_params={"deduplication_set_id": self.get_entity_id(deduplication_set)},
+        )
+
+    def retrieve(self, *args, **kwargs) -> None:
+        raise NotImplementedError("Retrieval of Image objects is not supported.")

src/country_workspace/contrib/dedup_engine/adapters/mixins.py

@@ -0,0 +1,44 @@
+from typing import Any, Type
+from urllib.parse import urljoin
+from uuid import UUID
+
+from country_workspace.contrib.dedup_engine.models import DeduplicationSet, Image
+from country_workspace.contrib.dedup_engine.types import TModel
+
+
+class URLMixin:
+    def prepare_url(self, path: str, **kwargs) -> str:
+        try:
+            formatted_path = path.format(**kwargs)
+        except KeyError as e:
+            raise ValueError(f"Missing placeholder '{e.args[0]}' in kwargs for path: '{path}'")
+        return urljoin(self.endpoints.base, formatted_path)
+
+
+class ValidationMixin:
+    @staticmethod
+    def get_entity_id(entity: DeduplicationSet | Image | UUID, id_field: str = "id") -> UUID:
+        match entity:
+            case UUID():
+                return entity
+            case _ if isinstance(entity, (DeduplicationSet, Image)):
+                try:
+                    return getattr(entity, id_field)
+                except AttributeError:
+                    raise AttributeError(f"'{type(entity).__name__}' does not have '{id_field}' attribute")
+            case _:
+                raise TypeError(
+                    f"Invalid type for entity: {type(entity).__name__}. Expected UUID, DeduplicationSet, or Image."
+                )
+
+    @staticmethod
+    def validate_data(data: Any, model_class: Type[TModel]) -> TModel:
+        match data:
+            case model_class():
+                return data
+            case dict():
+                return model_class.model_validate(data)
+            case _:
+                raise TypeError(
+                    f"Expected data to be of type {model_class.__name__} or dict, but got {type(data).__name__}"
+                )

src/country_workspace/contrib/dedup_engine/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class Config(AppConfig):
+    name = __name__.rpartition(".")[0]
+    verbose_name = "Country Workspace | Dedup Engine Client"