Endpoint tests with Sumo data

.github/workflows/backend_sumo_prod.yml

@@ -0,0 +1,63 @@
+name: integration
+
+on:
+  push:
+  pull_request:
+    branches:
+      - main
+  release:
+    types:
+      - published
+  # workflow_dispatch:
+  # schedule:
+  #   - cron: "48 4 * * *"
+
+jobs:
+  sumo_prod:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: 🤖 Get shared key from Sumo
+        working-directory: ./backend_py/primary
+        env:
+          SHARED_KEY_SUMO_PROD: ${{ secrets.SHARED_KEY_DROGON_READ_PROD }}
+        run: |
+          if [ ${#SHARED_KEY_SUMO_PROD} -eq 0 ]; then
+            echo "Error: SHARED_KEY_SUMO_PROD is empty. Stopping the action."
+            exit 1
+          fi
+          mkdir ~/.sumo
+          echo $SHARED_KEY_SUMO_PROD > ~/.sumo/9e5443dd-3431-4690-9617-31eed61cb55a.sharedkey
+
+      - name: 🐍 Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+          cache: pip
+
+      - name: 📦 Install poetry and dependencies
+        working-directory: ./backend_py/primary
+        run: |
+          pip install --upgrade pip
+          pip install poetry
+          poetry config virtualenvs.create false
+          poetry lock --check --no-update  # Check lock file is consistent with pyproject.toml
+          poetry install --with dev
+
+      - name: 🤖 Run tests
+        working-directory: ./backend_py/primary
+        env:
+          WEBVIZ_CLIENT_SECRET: 0
+          WEBVIZ_SMDA_SUBSCRIPTION_KEY: 0
+          WEBVIZ_SMDA_RESOURCE_SCOPE: 0
+          WEBVIZ_VDS_HOST_ADDRESS: 0
+          WEBVIZ_ENTERPRISE_SUBSCRIPTION_KEY: 0
+          WEBVIZ_SSDL_RESOURCE_SCOPE: 0
+          WEBVIZ_SUMU_ENV: prod
+        run: |
+          pytest -s --timeout=300 ./tests/integration

.github/workflows/webviz.yml

@@ -106,7 +106,7 @@ jobs:
           black --check primary/ tests/
           pylint primary/ tests/
           bandit --recursive primary/
-          mypy primary/ tests/
+          mypy primary/
 
       - name: 🤖 Run tests
         working-directory: ./backend_py/primary

backend_py/primary/primary/main.py

@@ -14,7 +14,7 @@
 from primary.middleware.add_process_time_to_server_timing_middleware import AddProcessTimeToServerTimingMiddleware
 from primary.routers.correlations.router import router as correlations_router
 from primary.routers.dev.router import router as dev_router
-from primary.routers.explore import router as explore_router
+from primary.routers.explore.router import router as explore_router
 from primary.routers.general import router as general_router
 from primary.routers.graph.router import router as graph_router
 from primary.routers.grid3d.router import router as grid3d_router

backend_py/primary/primary/routers/explore.py → backend_py/primary/primary/routers/explore/router.py

@@ -8,43 +8,21 @@
 from primary.services.sumo_access.sumo_inspector import SumoInspector
 from primary.services.utils.authenticated_user import AuthenticatedUser
 
-router = APIRouter()
-
-
-class FieldInfo(BaseModel):
-    field_identifier: str
-
-
-class CaseInfo(BaseModel):
-    uuid: str
-    name: str
-    status: str
-    user: str
+from . import schemas
 
-
-class EnsembleInfo(BaseModel):
-    name: str
-    realization_count: int
-
-
-class EnsembleDetails(BaseModel):
-    name: str
-    field_identifier: str
-    case_name: str
-    case_uuid: str
-    realizations: Sequence[int]
+router = APIRouter()
 
 
 @router.get("/fields")
 async def get_fields(
     authenticated_user: AuthenticatedUser = Depends(AuthHelper.get_authenticated_user),
-) -> List[FieldInfo]:
+) -> List[schemas.FieldInfo]:
     """
     Get list of fields
     """
     sumo_inspector = SumoInspector(authenticated_user.get_sumo_access_token())
     field_ident_arr = await sumo_inspector.get_fields_async()
-    ret_arr = [FieldInfo(field_identifier=field_ident.identifier) for field_ident in field_ident_arr]
+    ret_arr = [schemas.FieldInfo(field_identifier=field_ident.identifier) for field_ident in field_ident_arr]
 
     return ret_arr
 
@@ -53,14 +31,14 @@ async def get_fields(
 async def get_cases(
     authenticated_user: AuthenticatedUser = Depends(AuthHelper.get_authenticated_user),
     field_identifier: str = Query(description="Field identifier"),
-) -> List[CaseInfo]:
+) -> List[schemas.CaseInfo]:
     """Get list of cases for specified field"""
     sumo_inspector = SumoInspector(authenticated_user.get_sumo_access_token())
     case_info_arr = await sumo_inspector.get_cases_async(field_identifier=field_identifier)
 
-    ret_arr: List[CaseInfo] = []
+    ret_arr: List[schemas.CaseInfo] = []
 
-    ret_arr = [CaseInfo(uuid=ci.uuid, name=ci.name, status=ci.status, user=ci.user) for ci in case_info_arr]
+    ret_arr = [schemas.CaseInfo(uuid=ci.uuid, name=ci.name, status=ci.status, user=ci.user) for ci in case_info_arr]
 
     return ret_arr
 
@@ -69,22 +47,20 @@ async def get_cases(
 async def get_ensembles(
     authenticated_user: AuthenticatedUser = Depends(AuthHelper.get_authenticated_user),
     case_uuid: str = Path(description="Sumo case uuid"),
-) -> List[EnsembleInfo]:
+) -> List[schemas.EnsembleInfo]:
     """Get list of ensembles for a case"""
     case_inspector = CaseInspector.from_case_uuid(authenticated_user.get_sumo_access_token(), case_uuid)
     iteration_info_arr = await case_inspector.get_iterations_async()
 
-    print(iteration_info_arr)
-
-    return [EnsembleInfo(name=it.name, realization_count=it.realization_count) for it in iteration_info_arr]
+    return [schemas.EnsembleInfo(name=it.name, realization_count=it.realization_count) for it in iteration_info_arr]
 
 
 @router.get("/cases/{case_uuid}/ensembles/{ensemble_name}")
 async def get_ensemble_details(
     authenticated_user: AuthenticatedUser = Depends(AuthHelper.get_authenticated_user),
     case_uuid: str = Path(description="Sumo case uuid"),
     ensemble_name: str = Path(description="Ensemble name"),
-) -> EnsembleDetails:
+) -> schemas.EnsembleDetails:
     """Get more detailed information for an ensemble"""
 
     case_inspector = CaseInspector.from_case_uuid(authenticated_user.get_sumo_access_token(), case_uuid)
@@ -95,7 +71,7 @@ async def get_ensemble_details(
     if len(field_identifiers) != 1:
         raise NotImplementedError("Multiple field identifiers not supported")
 
-    return EnsembleDetails(
+    return schemas.EnsembleDetails(
         name=ensemble_name,
         case_name=case_name,
         case_uuid=case_uuid,

backend_py/primary/primary/routers/explore/schemas.py

@@ -0,0 +1,27 @@
+from typing import List, Sequence
+
+from pydantic import BaseModel
+
+
+class FieldInfo(BaseModel):
+    field_identifier: str
+
+
+class CaseInfo(BaseModel):
+    uuid: str
+    name: str
+    status: str
+    user: str
+
+
+class EnsembleInfo(BaseModel):
+    name: str
+    realization_count: int
+
+
+class EnsembleDetails(BaseModel):
+    name: str
+    field_identifier: str
+    case_name: str
+    case_uuid: str
+    realizations: Sequence[int]

backend_py/primary/primary/services/sumo_access/_helpers.py

@@ -11,7 +11,10 @@
 
 
 def create_sumo_client(access_token: str) -> SumoClient:
-    sumo_client = SumoClient(env=config.SUMO_ENV, token=access_token, interactive=False)
+    if access_token == "DUMMY_TOKEN_FOR_TESTING":  # nosec bandit B105
+        sumo_client = SumoClient(env=config.SUMO_ENV, interactive=False)
+    else:
+        sumo_client = SumoClient(env=config.SUMO_ENV, token=access_token, interactive=False)
     return sumo_client
 
 

backend_py/primary/pyproject.toml

@@ -32,11 +32,13 @@ server_schemas = {path = "../libs/server_schemas", develop = true}
 [tool.poetry.group.dev.dependencies]
 black = "^22.12.0"
 pylint = "^2.15.10"
-pytest = "^7.2.1"
+pytest = "^8.3.2"
 mypy = "^1.9.0"
 bandit = "^1.7.5"
 types-requests = "^2.31.0.1"
 types-redis = "^4.6.0"
+pytest-timeout = "^2.3.1"
+pytest-asyncio = "^0.24.0"
 
 
 [build-system]
@@ -63,5 +65,6 @@ disallow_untyped_defs = true
 [tool.pytest.ini_options]
 pythonpath = ["."]
 filterwarnings = "ignore::DeprecationWarning:pkg_resources"
-
+asyncio_mode="auto"
+asyncio_default_fixture_loop_scope="session"
 

backend_py/primary/tests/integration/conftest.py

@@ -0,0 +1,43 @@
+import os
+from dataclasses import dataclass
+
+import pytest
+
+from primary.services.utils.authenticated_user import AuthenticatedUser, AccessTokens
+
+
+@dataclass
+class SumoTestEnsemble:
+    field_identifier: str
+    case_uuid: str
+    case_name: str
+    ensemble_name: str
+
+
+@pytest.fixture(name="sumo_test_ensemble_ahm", scope="session")
+def fixture_sumo_test_ensemble_ahm() -> SumoTestEnsemble:
+    return SumoTestEnsemble(
+        field_identifier="DROGON",
+        case_name="webviz_ahm_case",
+        case_uuid="485041ce-ad72-48a3-ac8c-484c0ed95cf8",
+        ensemble_name="iter-0",
+    )
+
+
+@pytest.fixture(name="sumo_test_ensemble_design", scope="session")
+def fixture_sumo_test_ensemble_design() -> SumoTestEnsemble:
+    return SumoTestEnsemble(
+        field_identifier="DROGON",
+        case_name="01_drogon_design",
+        case_uuid="b89873c8-6f4d-40e5-978c-afc47beb2a26",
+        ensemble_name="iter-0",
+    )
+
+
+@pytest.fixture(name="test_user", scope="session")
+def fixture_test_user() -> AuthenticatedUser:
+    token = "DUMMY_TOKEN_FOR_TESTING"
+    tokens = AccessTokens(
+        sumo_access_token=token, graph_access_token=None, smda_access_token=None, ssdl_access_token=None
+    )
+    return AuthenticatedUser(user_id="test_user", username="test_user", access_tokens=tokens)