updates tests

.github/workflows/test.yml

@@ -142,12 +142,15 @@ jobs:
               -e CACHE_URL=${CACHE_URL} \
               -e SECRET_KEY=super-secret-key-just-for-testing \
               -e HOPE_API_URL="https://dev-hope.unitst.org/api/rest/" \
+              -e HOPE_API_TOKEN=${{ secrets.HOPE_API_TOKEN }} \
+              -e AURORA_API_URL="https://dev-hope.unitst.org/api/rest/" \
+              -e AURORA_API_TOKEN=${{ secrets.AURORA_API_TOKEN }} \
               -v "./output/:/app/output" \
               -v "./src/:/app/src" \
               -v "./tests:/app/tests" \
               -v "./pytest.ini:/app/pytest.ini" \
               -t ${{env.IMAGE}} \
-              pytest tests/ --selenium -n 4 -v --maxfail=5 --migrations --cov-report xml:./output/coverage.xml
+              pytest tests/ --selenium -n auto -v --maxfail=5 --migrations --cov-report xml:./output/coverage.xml
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4

pyproject.toml

@@ -93,6 +93,7 @@ dev-dependencies = [
     "pytest-echo>=1.7.3",
     "pytest-factoryboy>=2.7.0",
     "pytest-mock>=3.14.0",
+    "pytest-recording>=0.13.2",
     "pytest-selenium>=4.1.0",
     "pytest-xdist>=3.6.1",
     "pytest>=8.2.2",

pytest.ini

@@ -17,7 +17,7 @@ addopts =
         --cov country_workspace
         --cov-config=tests/.coveragerc
         --cov-report html
-
+        --dist loadgroup
 
 markers =
     selenium

src/country_workspace/contrib/hope/remotes/country.py

@@ -1,10 +1,11 @@
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 
 from django import forms
 
 from hope_flex_fields.attributes.abstract import AbstractAttributeHandler, AttributeHandlerConfig
 
 if TYPE_CHECKING:
+    from hope_flex_fields.models import FlexField
     from hope_flex_fields.types import Json
 
 
@@ -23,7 +24,7 @@ class CountryAttributeHandler(AbstractAttributeHandler):
     def set(self, value: "Json"):
         pass
 
-    def get(self) -> "Json":
+    def get(self, instance: "Optional[FlexField]" = None) -> "Json":
         from country_workspace.contrib.hope.client import HopeClient
 
         client = HopeClient()

src/country_workspace/contrib/hope/sync/office.py

@@ -1,5 +1,7 @@
 from typing import Optional
 
+from django.core.cache import cache
+
 from hope_flex_fields.models import DataChecker
 
 from country_workspace.models import Office, Program, SyncLog
@@ -11,24 +13,25 @@
 def sync_offices() -> dict[str, int]:
     client = HopeClient()
     totals = {"add": 0, "upd": 0}
-    for i, record in enumerate(client.get("business_areas")):
-        if record["active"]:
-            __, created = Office.objects.get_or_create(
-                hope_id=record["id"],
-                defaults={
-                    "name": record["name"],
-                    "slug": record["slug"],
-                    "code": record["code"],
-                    "active": record["active"],
-                    "long_name": record["long_name"],
-                },
-            )
-            if created:
-                totals["add"] += 1
-            else:
-                totals["upd"] += 1
-    SyncLog.objects.register_sync(Office)
-    return totals
+    with cache.lock("sync-offices"):
+        for i, record in enumerate(client.get("business_areas")):
+            if record["active"]:
+                __, created = Office.objects.get_or_create(
+                    hope_id=record["id"],
+                    defaults={
+                        "name": record["name"],
+                        "slug": record["slug"],
+                        "code": record["code"],
+                        "active": record["active"],
+                        "long_name": record["long_name"],
+                    },
+                )
+                if created:
+                    totals["add"] += 1
+                else:
+                    totals["upd"] += 1
+        SyncLog.objects.register_sync(Office)
+        return totals
 
 
 def sync_programs(limit_to_office: "Optional[Office]" = None) -> dict[str, int]:
@@ -38,34 +41,35 @@ def sync_programs(limit_to_office: "Optional[Office]" = None) -> dict[str, int]:
     totals = {"add": 0, "upd": 0}
     if limit_to_office:
         office = limit_to_office
-    for i, record in enumerate(client.get("programs")):
-        try:
-            if limit_to_office and record["business_area_code"] != office.code:
-                continue
-            else:
-                office = Office.objects.get(code=record["business_area_code"])
-            if record["status"] not in [Program.ACTIVE, Program.DRAFT]:
-                continue
-            p, created = Program.objects.get_or_create(
-                hope_id=record["id"],
-                defaults={
-                    "name": record["name"],
-                    "code": record["programme_code"],
-                    "status": record["status"],
-                    "sector": record["sector"],
-                    "country_office": office,
-                },
-            )
-            if created:
-                totals["add"] += 1
-                p.household_checker = hh_chk
-                p.individual_checker = ind_chk
-                p.save()
-            else:
-                totals["upd"] += 1
-        except Office.DoesNotExist:
-            pass
-    SyncLog.objects.register_sync(Program)
+    with cache.lock("sync-programs"):
+        for i, record in enumerate(client.get("programs")):
+            try:
+                if limit_to_office and record["business_area_code"] != office.code:
+                    continue
+                else:
+                    office = Office.objects.get(code=record["business_area_code"])
+                if record["status"] not in [Program.ACTIVE, Program.DRAFT]:
+                    continue
+                p, created = Program.objects.get_or_create(
+                    hope_id=record["id"],
+                    defaults={
+                        "name": record["name"],
+                        "code": record["programme_code"],
+                        "status": record["status"],
+                        "sector": record["sector"],
+                        "country_office": office,
+                    },
+                )
+                if created:
+                    totals["add"] += 1
+                    p.household_checker = hh_chk
+                    p.individual_checker = ind_chk
+                    p.save()
+                else:
+                    totals["upd"] += 1
+            except Office.DoesNotExist:
+                pass
+        SyncLog.objects.register_sync(Program)
     return totals
 
 

src/country_workspace/versioning/management/manager.py

@@ -75,12 +75,6 @@ def force_apply(self):
         for entry in self.existing:
             if entry.name not in self.applied:
                 self._process_file(entry)
-                # funcs = get_funcs(entry, direction="forward")
-                # for func in funcs:
-                #     try:
-                #         func()
-                #     except Exception as e:
-                #         raise ScriptException(f"Error executing {entry.stem}.{func.__name__}") from e
 
     def forward(
         self, to_num=None, fake: bool = False, out=sys.stdout

tests/conftest.py

@@ -4,7 +4,6 @@
 
 import pytest
 import responses
-from responses import _recorder
 
 here = Path(__file__).parent
 sys.path.insert(0, str(here / "../src"))
@@ -49,9 +48,9 @@ def pytest_configure(config):
     os.environ["CELERY_TASK_ALWAYS_EAGER"] = "1"
     os.environ["CELERY_TASK_STORE_EAGER_RESULT"] = "1"
     os.environ["SECURE_HSTS_PRELOAD"] = "0"
-    os.environ["AURORA_API_URL"] = "https://aurora.io/api/"
+    # os.environ["AURORA_API_URL"] = "https://aurora.io/api/"
     os.environ["AURORA_API_TOKEN"] = "aurora_token"
-    os.environ["HOPE_API_URL"] = "https://dev-hope.unitst.org/api/rest/"
+    # os.environ["HOPE_API_URL"] = "https://dev-hope.unitst.org/api/rest/"
     os.environ["HOPE_API_TOKEN"] = "kugiugiuygiuygiuygiuhgiuhgiuhgiugiu"
 
     os.environ["SECRET_KEY"] = "kugiugiuygiuygiuygiuhgiuhgiuhgiugiu"
@@ -62,10 +61,10 @@ def pytest_configure(config):
     import django
     from django.conf import settings
 
-    settings.AURORA_API_URL = os.environ["AURORA_API_URL"]
-    settings.AURORA_API_TOKEN = os.environ["AURORA_API_TOKEN"]
-    settings.HOPE_API_URL = "https://dev-hope.unitst.org/api/rest/"
-    settings.HOPE_API_TOKEN = os.environ["HOPE_API_TOKEN"]
+    # settings.AURORA_API_URL = os.environ["AURORA_API_URL"]
+    # settings.AURORA_API_TOKEN = os.environ["AURORA_API_TOKEN"]
+    # settings.HOPE_API_URL = "https://dev-hope.unitst.org/api/rest/"
+    # settings.HOPE_API_TOKEN = os.environ["HOPE_API_TOKEN"]
 
     settings.ALLOWED_HOSTS = ["127.0.0.1", "localhost"]
     settings.SIGNING_BACKEND = "testutils.signers.PlainSigner"
@@ -86,8 +85,6 @@ def setup(db, worker_id, settings):
     from testutils.factories import GroupFactory
 
     if worker_id != "master":
-        settings.CACHES["default"]["LOCATION"] = f"redis://localhost:6379/{worker_id}"
-        settings.CELERY_BROKER_URL = f"redis://localhost:6379/1{worker_id}"
         from country_workspace.cache.manager import cache_manager
 
         cache_manager.prefix = f"cache{worker_id}"
@@ -148,11 +145,9 @@ def active_marks(request):
 
 @pytest.fixture()
 def force_migrated_records(request, active_marks):
-    from country_workspace.models import SyncLog
     from country_workspace.versioning.management.manager import Manager
 
     Manager().force_apply()
-    _recorder.record(file_path=Path(__file__).parent / "r_sync_refresh.yaml")(lambda: SyncLog.objects.refresh())()
 
 
 @pytest.fixture()
@@ -171,3 +166,11 @@ def individual_checker(request, active_marks):
     from country_workspace.contrib.hope.constants import INDIVIDUAL_CHECKER_NAME
 
     return DataCheckerFactory(name=INDIVIDUAL_CHECKER_NAME)
+
+
+@pytest.fixture(scope="module")
+def vcr_config():
+    return {
+        "filter_headers": ["authorization"],
+        "cassette_library_dir": str(Path(__file__).parent / "extras/cassettes"),
+    }

tests/contrib/aurora/test_client.py

@@ -11,15 +11,12 @@
 from country_workspace.exceptions import RemoteError
 
 
-def tests_aurora_client_successfully(mock_vcr, mock_aurora_data):
+@pytest.mark.vcr(use_cassette="sync_aurora_4pages.yaml")
+def tests_aurora_client_successfully(mock_aurora_data):
     aurora_client = AuroraClient()
-    with patch("requests.get", wraps=requests.get) as mock_get:
-        with override_config(AURORA_API_URL=settings.AURORA_API_URL):
-            with mock_vcr.use_cassette(mock_aurora_data["cassette_name"]):
-                records = list(aurora_client.get("record"))
-                assert all(isinstance(record, dict) for record in records)
-                assert len(records) == mock_aurora_data["pages"] * mock_aurora_data["records_per_page"]
-                assert mock_get.call_count == mock_aurora_data["pages"]
+    with override_config(AURORA_API_URL=settings.AURORA_API_URL):
+        records = list(aurora_client.get("record"))
+        assert all(isinstance(record, dict) for record in records)
 
 
 @pytest.mark.parametrize(