chg ! tests

pyproject.toml

@@ -83,6 +83,7 @@ dev-dependencies = [
     "pytest-django>=4.8.0",
     "pytest-echo>=1.7.3",
     "pytest-factoryboy>=2.7.0",
+    "pytest-mock>=3.14.0",
     "pytest-selenium>=4.1.0",
     "pytest-xdist>=3.6.1",
     "pytest>=8.2.2",

src/country_workspace/admin/job.py

@@ -16,7 +16,6 @@ class AsyncJobAdmin(CeleryTaskModelAdmin, BaseModelAdmin):
         "task_status",
     )
     list_filter = ("type", ("program", AutoCompleteFilter))
-    ordering = ("id",)
 
     def has_add_permission(self, request):
         return False

src/country_workspace/config/__init__.py

@@ -53,7 +53,7 @@ class Group(Enum):
         setting("allowed-hosts"),
     ),
     "AURORA_API_TOKEN": (str, "", "", True, "Aurora API token"),
-    "AURORA_API_URL": (str, "", "", True, "Aurora API url"),
+    "AURORA_API_URL": (str, "", "", False, "Aurora API url"),
     "CACHE_URL": (str, "", "redis://localhost:6379/0", True, setting("cache-url")),
     "CELERY_BROKER_URL": (
         str,

src/country_workspace/contrib/aurora/client.py

@@ -1,9 +1,12 @@
+from json import JSONDecodeError
 from typing import Any, Generator
 from urllib.parse import urljoin
 
 import requests
 from constance import config
 
+from country_workspace.exceptions import RemoteError
+
 
 class AuroraClient:
     """
@@ -49,14 +52,23 @@ def get(self, path: str) -> Generator[dict[str, Any], None, None]:
             dict[str, Any]: Individual records from the API.
 
         Raises:
-            Exception: If the API response has a status code other than 200.
+            RemoteError: If the API response has a non-200 status code,
+                         if there's an issue with the network request,
+                         or if the response contains invalid JSON.
         """
         url = self._get_url(path)
         while url:
-            ret = requests.get(url, headers={"Authorization": f"Token {self.token}"}, timeout=10)
-            if ret.status_code != 200:
-                raise Exception(f"Error {ret.status_code} fetching {url}")
-            data = ret.json()
+            try:
+                ret = requests.get(url, headers={"Authorization": f"Token {self.token}"}, timeout=10)
+                if ret.status_code != 200:
+                    raise RemoteError(f"Error {ret.status_code} fetching {url}")
+            except requests.RequestException:
+                raise RemoteError(f"Remote Error fetching {url}")
+
+            try:
+                data = ret.json()
+            except JSONDecodeError:
+                raise RemoteError(f"Wrong JSON response fetching {url}")
 
             for record in data["results"]:
                 yield record

src/country_workspace/workspaces/admin/program.py

@@ -2,6 +2,7 @@
 
 from django import forms
 from django.conf import settings
+from django.contrib import messages
 from django.contrib.admin import register
 from django.db.models import QuerySet
 from django.db.transaction import atomic
@@ -262,7 +263,7 @@ def import_file_updates(self, request: HttpRequest, pk: str) -> "HttpResponse":
         return render(request, "workspace/actions/bulk_update_import.html", context)
 
     @button(label=_("Import from Aurora"))
-    def import_aurora(self, request: HttpRequest, pk: str) -> "HttpResponse":
+    def import_aurora(self, request: HttpRequest, pk: str) -> HttpResponse:
         context = self.get_common_context(request, pk, title="Import from Aurora")
         program: CountryProgram = context["original"]
         context["selected_program"] = context["original"]
@@ -282,7 +283,7 @@ def import_aurora(self, request: HttpRequest, pk: str) -> "HttpResponse":
                     _("The import task from Aurora has been successfully queued. Asynchronous task ID: {0}.").format(
                         j.curr_async_result_id
                     ),
-                    level="success",
+                    level=messages.SUCCESS,
                 )
                 return HttpResponseRedirect(self.get_changelist_url())
         else:

tests/conftest.py

@@ -50,15 +50,16 @@ def pytest_configure(config):
     os.environ["CELERY_TASK_ALWAYS_EAGER"] = "1"
     os.environ["SECURE_HSTS_PRELOAD"] = "0"
     os.environ["EXTRA_APPS"] = "country_workspace.contrib.hope"
-    os.environ["HOPE_API_URL"] = "https://dev-hope.unitst.org/api/rest/"
     os.environ["HOPE_API_TOKEN"] = "kugiugiuygiuygiuygiuhgiuhgiuhgiugiu"
+    os.environ["HOPE_API_URL"] = "https://dev-hope.unitst.org/api/rest/"
     # os.environ["SECRET_KEY"] = "kugiugiuygiuygiuygiuhgiuhgiuhgiugiu"
 
     os.environ["LOGGING_LEVEL"] = "CRITICAL"
     import django
     from django.conf import settings
 
     settings.ALLOWED_HOSTS = ["127.0.0.1", "localhost"]
+    settings.AURORA_API_URL = "https://uni-hope-ukr-sr-dev.unitst.org/api/"
     settings.SIGNING_BACKEND = "testutils.signers.PlainSigner"
     settings.SECRET_KEY = "kugiugiuygiuygiuygiuhgiuhgiuhgiugiu"
     settings.CSRF_TRUSTED_ORIGINS = [

tests/contrib/aurora/conftest.py

@@ -0,0 +1,100 @@
+from pathlib import Path
+from typing import Any
+from unittest.mock import MagicMock
+
+import pytest
+import vcr
+from extras.testutils.factories import (
+    AsyncJobFactory,
+    BatchFactory,
+    HouseholdFactory,
+    IndividualFactory,
+    ProgramFactory,
+    UserFactory,
+)
+from pytest_mock import MockerFixture
+
+from country_workspace.contrib.aurora.client import AuroraClient
+from country_workspace.models import AsyncJob
+
+
+@pytest.fixture
+def mock_vcr() -> vcr.VCR:
+    return vcr.VCR(
+        filter_headers=["authorization"],
+        cassette_library_dir=str(Path(__file__).parent.parent.parent / "extras/cassettes"),
+        record_mode=vcr.record_mode.RecordMode.ONCE,
+        match_on=("path",),
+    )
+
+
+@pytest.fixture
+def mock_aurora_data() -> dict[str, Any]:
+    return {
+        "cassette_name": "sync_aurora_4pages.yaml",
+        "pages": 4,
+        "records_per_page": 10,
+        "households": 1,
+        "individuals": 2,
+        "results": [
+            {
+                "fields": {
+                    "household": [{"field_hh1": "value_hh1"}],
+                    "individuals": [
+                        {"field_i1": "value_i1"},
+                        {"field_i2": "value_i2"},
+                    ],
+                }
+            }
+        ],
+        "form_cleaned_data": {
+            "batch_name": "Batch 1",
+            "household_name_column": "family_name",
+        },
+        "imported_by_id": 1,
+    }
+
+
+@pytest.fixture
+def mock_aurora_client(mocker: MockerFixture, mock_aurora_data: dict[str, Any]) -> MagicMock:
+    client = mocker.MagicMock(spec=AuroraClient)
+    client.get.return_value = mock_aurora_data["results"]
+    return client
+
+
+@pytest.fixture
+def program():
+    return ProgramFactory()
+
+
+@pytest.fixture
+def batch(program):
+    return BatchFactory(program=program)
+
+
+@pytest.fixture
+def user():
+    return UserFactory()
+
+
+@pytest.fixture
+def job(mock_aurora_data, program, batch, user):
+    return AsyncJobFactory(
+        type=AsyncJob.JobType.AURORA_SYNC,
+        program=program,
+        batch=batch,
+        config={
+            **mock_aurora_data["form_cleaned_data"],
+            "imported_by_id": user.pk,
+        },
+    )
+
+
+@pytest.fixture
+def household(batch):
+    return HouseholdFactory(batch=batch)
+
+
+@pytest.fixture
+def individuals(batch, household):
+    return IndividualFactory.create_batch(2, batch=batch, household=household)

tests/contrib/aurora/test_aurora_sync.py

@@ -0,0 +1,150 @@
+from unittest.mock import patch
+
+import pytest
+
+from country_workspace.contrib.aurora.sync import (
+    _create_batch,
+    _create_household,
+    _create_individuals,
+    _update_household_name_from_individual,
+    sync_aurora_job,
+)
+from country_workspace.models import Batch, Household, Office, Program, User
+
+
+def test_create_batch_success(mock_aurora_data, job, user):
+    batch = _create_batch(job)
+    assert isinstance(batch, Batch)
+    assert isinstance(batch.country_office, Office)
+    assert isinstance(batch.program, Program)
+    assert isinstance(batch.imported_by, User)
+    assert batch.name == mock_aurora_data["form_cleaned_data"]["batch_name"]
+    assert batch.program == job.program
+    assert batch.country_office == job.program.country_office
+    assert batch.imported_by == user
+
+
+def test_create_household_success(mock_aurora_data, job):
+    fields = mock_aurora_data["results"][0]["fields"]["household"][0]
+    household = _create_household(job, fields)
+
+    assert isinstance(household, Household)
+    assert household.program == job.program
+    assert household.batch == job.batch
+    assert household.country_office == job.program.country_office
+    assert household.flex_fields == fields
+
+
+@pytest.mark.parametrize(
+    "data, expected_name_update",
+    [
+        (
+            {"relationship_to_head": "head", "family_name": "Head Of Household Name"},
+            "Head Of Household Name",
+        ),
+        (
+            {"relationship_to_head": "child", "family_name": "Child Name"},
+            None,
+        ),
+        (
+            {"relationship_to_head": "head"},
+            None,
+        ),
+        (
+            {},
+            None,
+        ),
+    ],
+    ids=[
+        "Head with name update",
+        "Non-head individual",
+        "Head without name",
+        "Empty individual data",
+    ],
+)
+def test_update_household_name_from_individual(mock_aurora_data, job, household, data, expected_name_update):
+    initial_name = household.name
+
+    individual_data = mock_aurora_data["results"][0]["fields"]["individuals"][0].copy()
+    individual_data.update(data)
+    _update_household_name_from_individual(job, household, individual_data)
+    household.refresh_from_db()
+
+    if expected_name_update:
+        assert household.name == expected_name_update
+    else:
+        assert household.name == initial_name
+
+
+@pytest.mark.parametrize(
+    "fields, expected_count",
+    [
+        (
+            [
+                {"given_name": "John", "family_name": "Doe", "relationship_to_head": "head"},
+                {"given_name": "Jane", "family_name": "Doe", "relationship_to_head": "spouse"},
+            ],
+            2,
+        ),
+        (
+            [],
+            0,
+        ),
+    ],
+    ids=["filled_fields", "empty_fields"],
+)
+def test_create_individuals(mock_aurora_data, job, household, fields, expected_count):
+    with (
+        patch("country_workspace.contrib.aurora.sync._update_household_name_from_individual") as mock_update_name,
+        patch(
+            "country_workspace.contrib.aurora.sync.clean_field_name", side_effect=lambda x: f"cleaned_{x}"
+        ) as mock_clean_field_name,
+    ):
+
+        individuals = _create_individuals(job, household, fields)
+
+        assert len(individuals) == expected_count
+
+        assert mock_update_name.call_count == expected_count
+        if expected_count > 0:
+
+            for individual in fields:
+                mock_update_name.assert_any_call(job, household, individual)
+
+            for individual, data in zip(individuals, fields):
+                assert individual.household_id == household.pk
+                assert individual.batch == job.batch
+                assert individual.name == data.get("given_name", "")
+                assert individual.flex_fields == {f"cleaned_{k}": v for k, v in data.items()}
+                mock_clean_field_name.assert_any_call("given_name")
+
+
+def test_sync_aurora_job_success(mock_aurora_client, mock_aurora_data, job, household, individuals):
+    with (
+        patch("country_workspace.contrib.aurora.sync.AuroraClient", return_value=mock_aurora_client),
+        patch("country_workspace.contrib.aurora.sync._create_batch", return_value=job.batch) as mock_create_batch,
+        patch(
+            "country_workspace.contrib.aurora.sync._create_household", return_value=household
+        ) as mock_create_household,
+        patch(
+            "country_workspace.contrib.aurora.sync._create_individuals", return_value=individuals
+        ) as mock_create_individuals,
+        patch.object(job, "save", wraps=job.save) as mock_save_job,
+    ):
+        mock_aurora_client.get.return_value = mock_aurora_data["results"]
+
+        result = sync_aurora_job(job)
+
+        mock_create_batch.assert_called_once_with(job)
+        assert mock_aurora_client.get.called
+        mock_create_household.assert_called_once_with(job, mock_aurora_data["results"][0]["fields"]["household"][0])
+        mock_create_individuals.assert_called_once_with(
+            job, household, mock_aurora_data["results"][0]["fields"]["individuals"]
+        )
+
+        assert mock_save_job.call_count == 1
+
+        assert result == {
+            "households": 1,
+            "individuals": len(individuals),
+        }