add ! hde client

pyproject.toml

@@ -50,6 +50,7 @@ dependencies = [
     "sentry-sdk>=2.7.1",
     "social-auth-app-django",
     "unicef-security>=1.5.1",
+    "pydantic>=2.10.1",
 ]
 [project.scripts]
 celery-monitor = "country_workspace.__monitor__:run"

src/country_workspace/config/__init__.py

@@ -100,6 +100,8 @@ class Group(Enum):
         "https://django-environ.readthedocs.io/en/latest/types.html#environ-env-db-url",
     ),
     "DEBUG": (bool, False, True, False, setting("debug")),
+    "DEDUP_ENGINE_API_TOKEN": (str, "", "", False, "Dedup engine API token"),
+    "DEDUP_ENGINE_API_URL": (str, "", "", False, "Dedup engine API url"),
     # "EMAIL_BACKEND": (
     #     str,
     #     "django.core.mail.backends.smtp.EmailBackend",

src/country_workspace/config/fragments/app.py

@@ -12,6 +12,10 @@
 HOPE_API_TOKEN = env("HOPE_API_TOKEN")
 HOPE_API_URL = env("HOPE_API_URL")
 
+DEDUP_ENGINE_API_TOKEN = env("DEDUP_ENGINE_API_TOKEN")
+DEDUP_ENGINE_API_URL = env("DEDUP_ENGINE_API_URL")
+
+
 HH_LOOKUPS = [
     "ResidenceStatus",
 ]

src/country_workspace/config/fragments/constance.py

@@ -1,6 +1,8 @@
 from country_workspace.config.fragments.app import (
     AURORA_API_TOKEN,
     AURORA_API_URL,
+    DEDUP_ENGINE_API_TOKEN,
+    DEDUP_ENGINE_API_URL,
     HOPE_API_TOKEN,
     HOPE_API_URL,
     NEW_USER_DEFAULT_GROUP,
@@ -52,6 +54,9 @@
     "AURORA_API_URL": (AURORA_API_URL, "Aurora API Server address", str),
     "HOPE_API_TOKEN": (HOPE_API_TOKEN, "HOPE API Access Token", "write_only_input"),
     "HOPE_API_URL": (HOPE_API_URL, "HOPE API Server address", str),
+    "DEDUP_ENGINE_API_TOKEN": (DEDUP_ENGINE_API_TOKEN, "Dedup engine API Access Token", "write_only_input"),
+    "DEDUP_ENGINE_API_URL": (DEDUP_ENGINE_API_URL, "Dedup engine API Server address", str),
+    "KOBO_API_URL": ("", "Kobo API Server address", str),
     "KOBO_API_TOKEN": ("", "Kobo API Access Token", "write_only_input"),
     "KOBO_API_URL": ("", "Kobo API Server address", str),
 }

src/country_workspace/config/settings.py

@@ -50,6 +50,7 @@
     "country_workspace.apps.Config",
     "country_workspace.workspaces.apps.Config",
     "country_workspace.contrib.aurora.apps.Config",
+    "country_workspace.contrib.dedup_engine.apps.Config",
     "country_workspace.versioning",
     *env("EXTRA_APPS"),
 )

src/country_workspace/contrib/dedup_engine/adapters/__init__.py

@@ -0,0 +1,3 @@
+from .deduplication_set import DeduplicationSetAdapter  # noqa
+from .duplicate import DuplicateAdapter  # noqa
+from .image import ImageAdapter  # noqa

src/country_workspace/contrib/dedup_engine/adapters/base.py

@@ -0,0 +1,25 @@
+from urllib.parse import urljoin
+from uuid import UUID
+
+from requests import Session
+
+from country_workspace.contrib.dedup_engine.models import DeduplicationSet, Image
+
+
+class BaseAdapter:
+    def __init__(self, session: Session, base_url: str, resource: str) -> None:
+        self.session = session
+        self.base_url = urljoin(base_url, f"{resource}/")
+
+    def prepare_url(self, *parts: str) -> str:
+        return urljoin(self.base_url, "/".join(parts))
+
+    @staticmethod
+    def get_entity_id(entity: DeduplicationSet | Image | UUID, id_field: str = "id") -> UUID:
+        if isinstance(entity, UUID):
+            return entity
+        elif isinstance(entity, (DeduplicationSet, Image)):
+            if not hasattr(entity, id_field):
+                raise AttributeError(f"Object of type {type(entity).__name__} does not have attribute '{id_field}'")
+            return getattr(entity, id_field)
+        raise TypeError(f"Expected DeduplicationSet, Image, or UUID, but got {type(entity).__name__}")

src/country_workspace/contrib/dedup_engine/adapters/deduplication_set.py

@@ -0,0 +1,43 @@
+from uuid import UUID
+
+from requests import Session
+
+from country_workspace.contrib.dedup_engine.adapters.base import BaseAdapter
+from country_workspace.contrib.dedup_engine.models import DeduplicationSet, DeduplicationSetCreate
+
+
+class DeduplicationSetAdapter(BaseAdapter):
+    def __init__(self, session: Session, base_url: str) -> None:
+        super().__init__(session, base_url, "deduplication_sets")
+
+    def create(self, data: DeduplicationSetCreate) -> DeduplicationSet:
+        url = self.base_url
+        response = self.session.post(url, json=data.model_dump(by_alias=True))
+        response.raise_for_status()
+        return DeduplicationSet(**response.json())
+
+    def destroy(self, data: DeduplicationSet | UUID) -> None:
+        deduplication_set_id = self.get_entity_id(data)
+        url = self.prepare_url(str(deduplication_set_id))
+        response = self.session.delete(url)
+        if response.status_code != 204:
+            response.raise_for_status()
+
+    def list(self) -> list[DeduplicationSet]:
+        response = self.session.get(self.base_url)
+        response.raise_for_status()
+        return [DeduplicationSet(**item) for item in response.json()]
+
+    def retrieve(self, data: DeduplicationSet | UUID) -> DeduplicationSet:
+        deduplication_set_id = self.get_entity_id(data)
+        url = self.prepare_url(str(deduplication_set_id))
+        response = self.session.get(url)
+        response.raise_for_status()
+        return DeduplicationSet(**response.json())
+
+    def process(self, data: DeduplicationSet | UUID) -> None:
+        deduplication_set_id = self.get_entity_id(data)
+        url = self.prepare_url(f"{deduplication_set_id}/process/")
+        response = self.session.post(url)
+        if response.status_code != 200:
+            response.raise_for_status()

src/country_workspace/contrib/dedup_engine/adapters/duplicate.py

@@ -0,0 +1,20 @@
+from urllib.parse import urljoin
+from uuid import UUID
+
+from requests import Session
+
+from country_workspace.contrib.dedup_engine.adapters.base import BaseAdapter
+from country_workspace.contrib.dedup_engine.models import DeduplicationSet, Duplicate
+
+
+class DuplicateAdapter(BaseAdapter):
+    def __init__(self, session: Session, base_url: str) -> None:
+        self.session = session
+        self.base_url = urljoin(base_url, "deduplication_sets/")
+
+    def list(self, data: DeduplicationSet | UUID) -> list[Duplicate]:
+        deduplication_set_id = self.get_entity_id(data)
+        url = self.prepare_url(f"{deduplication_set_id}/duplicates/")
+        response = self.session.get(url)
+        response.raise_for_status()
+        return [Duplicate(**item) for item in response.json()]

src/country_workspace/contrib/dedup_engine/adapters/image.py

@@ -0,0 +1,35 @@
+from urllib.parse import urljoin
+from uuid import UUID
+
+from requests import Session
+
+from country_workspace.contrib.dedup_engine.adapters.base import BaseAdapter
+from country_workspace.contrib.dedup_engine.models import DeduplicationSet, Image, ImageCreate
+
+
+class ImageAdapter(BaseAdapter):
+    def __init__(self, session: Session, base_url: str) -> None:
+        self.session = session
+        self.base_url = urljoin(base_url, "deduplication_sets/")
+
+    def list(self, data: DeduplicationSet | UUID) -> list[Image]:
+        deduplication_set_id = self.get_entity_id(data)
+        url = self.prepare_url(f"{deduplication_set_id}/images/")
+        response = self.session.get(url)
+        response.raise_for_status()
+        return [Image(**item) for item in response.json()]
+
+    def create(self, deduplication_set: DeduplicationSet | UUID, data: ImageCreate) -> Image:
+        deduplication_set_id = self.get_entity_id(deduplication_set)
+        url = self.prepare_url(f"{deduplication_set_id}/images/")
+        response = self.session.post(url, json=data.model_dump(by_alias=True))
+        response.raise_for_status()
+        return Image(**response.json())
+
+    def delete(self, deduplication_set: DeduplicationSet | UUID, image: Image | UUID) -> None:
+        deduplication_set_id = self.get_entity_id(deduplication_set)
+        image_id = self.get_entity_id(image)
+        url = self.prepare_url(f"{deduplication_set_id}/images/{image_id}/")
+        response = self.session.delete(url)
+        if response.status_code != 204:
+            response.raise_for_status()

src/country_workspace/contrib/dedup_engine/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class Config(AppConfig):
+    name = __name__.rpartition(".")[0]
+    verbose_name = "Country Workspace | Dedup Engine Client"

src/country_workspace/contrib/dedup_engine/client.py

@@ -0,0 +1,33 @@
+from requests import Session
+from requests.auth import AuthBase
+from requests.models import PreparedRequest
+
+from country_workspace.contrib.dedup_engine.adapters import DeduplicationSetAdapter, DuplicateAdapter, ImageAdapter
+
+
+class Auth(AuthBase):
+    def __init__(self, token: str) -> None:
+        self._auth_header = f"Token {token}"
+
+    def __call__(self, request: PreparedRequest) -> PreparedRequest:
+        request.headers["Authorization"] = self._auth_header
+        return request
+
+
+class HDEAPIClient:
+    def __init__(self, *, base_url: str, token: str) -> None:
+        self.base_url = base_url.rstrip("/")
+        self.session = Session()
+        self.session.auth = Auth(token)
+
+    @property
+    def deduplication_set(self) -> DeduplicationSetAdapter:
+        return DeduplicationSetAdapter(self.session, self.base_url)
+
+    @property
+    def duplicate(self) -> DuplicateAdapter:
+        return DuplicateAdapter(self.session, self.base_url)
+
+    @property
+    def image(self) -> ImageAdapter:
+        return ImageAdapter(self.session, self.base_url)

src/country_workspace/contrib/dedup_engine/exceptions.py

@@ -0,0 +1,4 @@
+class DedupEngineAPIBusinessError(Exception):
+    def __init__(self, detail: str):
+        self.detail = detail
+        super().__init__(f"Business logic error: {detail}")

src/country_workspace/contrib/dedup_engine/models/__init__.py

@@ -0,0 +1,3 @@
+from .deduplication_set import DeduplicationSet, DeduplicationSetCreate  # noqa
+from .duplicate import Duplicate  # noqa
+from .image import Image, ImageCreate  # noqa

src/country_workspace/contrib/dedup_engine/models/deduplication_set.py

@@ -0,0 +1,52 @@
+from datetime import datetime
+from enum import Enum
+from typing import Any
+from uuid import UUID
+
+from pydantic import BaseModel
+
+
+class DeduplicationSetStatus(Enum):
+    CLEAN = "Clean"
+    DIRTY = "Dirty"
+
+    @property
+    def label(self) -> str:
+        return self.value
+
+    @property
+    def description(self) -> str:
+        if self == DeduplicationSetStatus.CLEAN:
+            return "Deduplication set is created or already processed"
+        elif self == DeduplicationSetStatus.DIRTY:
+            return "Deduplication set needs processing"
+
+
+class DeduplicationSetConfig(BaseModel):
+    name: str | None = None
+    settings: dict[str, Any] | None = None
+
+
+class DeduplicationSetCreate(BaseModel):
+    reference_pk: str
+    name: str | None = None
+    description: str | None = None
+    notification_url: str | None = None
+
+
+class DeduplicationSet(DeduplicationSetCreate):
+    id: UUID
+    state: DeduplicationSetStatus
+    config: DeduplicationSetConfig | None = None
+    created_at: datetime
+    updated_at: datetime | None = None
+    external_system: str
+    created_by: int
+    updated_by: int | None = None
+
+    model_config = {
+        "json_encoders": {
+            datetime: lambda v: v.isoformat(),
+            DeduplicationSetStatus: lambda v: v.label,
+        }
+    }

src/country_workspace/contrib/dedup_engine/models/duplicate.py

@@ -0,0 +1,13 @@
+from uuid import UUID
+
+from pydantic import BaseModel
+
+
+class DuplicateReference(BaseModel):
+    reference_pk: UUID
+
+
+class Duplicate(BaseModel):
+    first: DuplicateReference
+    second: DuplicateReference
+    score: float

src/country_workspace/contrib/dedup_engine/models/image.py

@@ -0,0 +1,22 @@
+from datetime import datetime
+from uuid import UUID
+
+from pydantic import BaseModel
+
+
+class ImageCreate(BaseModel):
+    reference_pk: str  # perhaps better UUID, but need changes in the server
+    filename: str
+
+
+class Image(ImageCreate):
+    id: UUID
+    deduplication_set: UUID
+    created_by: int | None = None
+    created_at: datetime
+
+    model_config = {
+        "json_encoders": {
+            datetime: lambda v: v.isoformat(),
+        }
+    }