From 6ab56a804023debe4e15951a09d7bd5b3d705ee5 Mon Sep 17 00:00:00 2001 From: Hans Kallekleiv <16436291+HansKallekleiv@users.noreply.github.com> Date: Tue, 3 Sep 2024 09:44:57 +0200 Subject: [PATCH] wip --- backend_py/primary/primary/main.py | 2 +- .../primary/primary/routers/explore/router.py | 5 -- .../primary/routers/explore/schemas.py | 2 +- .../primary/routers/parameters/router.py | 58 +++++++++--- .../primary/routers/parameters/schemas.py | 45 +++++++++- .../primary/tests/integration/conftest.py | 13 ++- .../routers/explore/test_explore.py | 31 +++---- .../routers/parameters/test_parameters.py | 88 +++++++++++++++++++ .../test_get_realizations_vector_data.py | 12 +-- .../timeseries/test_get_vector_list.py | 4 +- 10 files changed, 215 insertions(+), 45 deletions(-) create mode 100644 backend_py/primary/tests/integration/routers/parameters/test_parameters.py diff --git a/backend_py/primary/primary/main.py b/backend_py/primary/primary/main.py index 441fd3e8a..62e35d58f 100644 --- a/backend_py/primary/primary/main.py +++ b/backend_py/primary/primary/main.py @@ -14,7 +14,7 @@ from primary.middleware.add_process_time_to_server_timing_middleware import AddProcessTimeToServerTimingMiddleware from primary.routers.correlations.router import router as correlations_router from primary.routers.dev.router import router as dev_router -from primary.routers.explore import router as explore_router +from primary.routers.explore.router import router as explore_router from primary.routers.general import router as general_router from primary.routers.graph.router import router as graph_router from primary.routers.grid3d.router import router as grid3d_router diff --git a/backend_py/primary/primary/routers/explore/router.py b/backend_py/primary/primary/routers/explore/router.py index decc337d8..e64b35411 100644 --- a/backend_py/primary/primary/routers/explore/router.py +++ b/backend_py/primary/primary/routers/explore/router.py @@ -13,9 +13,6 @@ router = APIRouter() - - - @router.get("/fields") async def get_fields( authenticated_user: AuthenticatedUser = Depends(AuthHelper.get_authenticated_user), @@ -55,8 +52,6 @@ async def get_ensembles( case_inspector = CaseInspector.from_case_uuid(authenticated_user.get_sumo_access_token(), case_uuid) iteration_info_arr = await case_inspector.get_iterations_async() - print(iteration_info_arr) - return [schemas.EnsembleInfo(name=it.name, realization_count=it.realization_count) for it in iteration_info_arr] diff --git a/backend_py/primary/primary/routers/explore/schemas.py b/backend_py/primary/primary/routers/explore/schemas.py index 4d61c069e..61adf457d 100644 --- a/backend_py/primary/primary/routers/explore/schemas.py +++ b/backend_py/primary/primary/routers/explore/schemas.py @@ -24,4 +24,4 @@ class EnsembleDetails(BaseModel): field_identifier: str case_name: str case_uuid: str - realizations: Sequence[int] \ No newline at end of file + realizations: Sequence[int] diff --git a/backend_py/primary/primary/routers/parameters/router.py b/backend_py/primary/primary/routers/parameters/router.py index 71c3882e1..92293512f 100644 --- a/backend_py/primary/primary/routers/parameters/router.py +++ b/backend_py/primary/primary/routers/parameters/router.py @@ -5,7 +5,6 @@ from primary.auth.auth_helper import AuthHelper from primary.services.sumo_access.parameter_access import ParameterAccess -from primary.services.sumo_access.parameter_types import EnsembleParameter, EnsembleSensitivity from primary.services.utils.authenticated_user import AuthenticatedUser from . import schemas @@ -53,16 +52,25 @@ async def get_parameter( case_uuid: str = Query(description="Sumo case uuid"), ensemble_name: str = Query(description="Ensemble name"), parameter_name: str = Query(description="Parameter name"), -) -> Optional[EnsembleParameter]: +) -> Optional[schemas.EnsembleParameter]: """Get a parameter in a given Sumo ensemble""" access = await ParameterAccess.from_case_uuid_async( authenticated_user.get_sumo_access_token(), case_uuid, ensemble_name ) - parameters = (await access.get_parameters_and_sensitivities()).parameters - for parameter in parameters: + parameters_and_sensitivities = await access.get_parameters_and_sensitivities() + for parameter in parameters_and_sensitivities.parameters: if parameter.name == parameter_name: - return parameter + return schemas.EnsembleParameter( + name=parameter.name, + is_logarithmic=parameter.is_logarithmic, + is_numerical=parameter.is_numerical, + is_constant=parameter.is_constant, + group_name=parameter.group_name, + descriptive_name=parameter.descriptive_name, + realizations=parameter.realizations, + values=parameter.values, + ) return None @@ -71,12 +79,27 @@ async def get_parameters( authenticated_user: AuthenticatedUser = Depends(AuthHelper.get_authenticated_user), case_uuid: str = Query(description="Sumo case uuid"), ensemble_name: str = Query(description="Ensemble name"), -) -> List[EnsembleParameter]: +) -> List[schemas.EnsembleParameter]: access = await ParameterAccess.from_case_uuid_async( authenticated_user.get_sumo_access_token(), case_uuid, ensemble_name ) - parameters = (await access.get_parameters_and_sensitivities()).parameters - return [parameter for parameter in parameters] + parameters_and_sensitivities = await access.get_parameters_and_sensitivities() + parameters_ret_arr: List[schemas.EnsembleParameter] = [] + for parameter in parameters_and_sensitivities.parameters: + parameters_ret_arr.append( + schemas.EnsembleParameter( + name=parameter.name, + is_logarithmic=parameter.is_logarithmic, + is_numerical=parameter.is_numerical, + is_constant=parameter.is_constant, + group_name=parameter.group_name, + descriptive_name=parameter.descriptive_name, + realizations=parameter.realizations, + values=parameter.values, + ) + ) + + return parameters_ret_arr @router.get("/is_sensitivity_run/") @@ -99,12 +122,25 @@ async def get_sensitivities( authenticated_user: AuthenticatedUser = Depends(AuthHelper.get_authenticated_user), case_uuid: str = Query(description="Sumo case uuid"), ensemble_name: str = Query(description="Ensemble name"), -) -> List[EnsembleSensitivity]: +) -> List[schemas.EnsembleSensitivity]: """Get sensitivities in a given Sumo ensemble""" access = await ParameterAccess.from_case_uuid_async( authenticated_user.get_sumo_access_token(), case_uuid, ensemble_name ) - sensitivities = (await access.get_parameters_and_sensitivities()).sensitivities - return sensitivities if sensitivities else [] + parameters_and_sensitivities = (await access.get_parameters_and_sensitivities()) + sensitivities_ret_arr: List[schemas.EnsembleSensitivity] = [] + for sensitivity in parameters_and_sensitivities.sensitivities: + sensitivities_ret_arr.append( + schemas.EnsembleSensitivity( + name=sensitivity.name, + type=sensitivity.type, + cases=[ + schemas.EnsembleSensitivityCase(name=case.name, realizations=case.realizations) + for case in sensitivity.cases + ], + ) + ) + + return sensitivities_ret_arr diff --git a/backend_py/primary/primary/routers/parameters/schemas.py b/backend_py/primary/primary/routers/parameters/schemas.py index 20ed66de0..f6cbd8b81 100644 --- a/backend_py/primary/primary/routers/parameters/schemas.py +++ b/backend_py/primary/primary/routers/parameters/schemas.py @@ -1,4 +1,5 @@ -from typing import Optional +from enum import Enum +from typing import Optional, List, Union from pydantic import BaseModel @@ -8,3 +9,45 @@ class EnsembleParameterDescription(BaseModel): group_name: Optional[str] = None descriptive_name: Optional[str] = None is_numerical: bool + + +class EnsembleParameter(BaseModel): + """Description/data for a single parameter in an ensemble""" + + name: str + is_logarithmic: bool + is_numerical: bool + is_constant: bool # all values are equal + group_name: Optional[str] = None + descriptive_name: Optional[str] = None + realizations: List[int] + values: Union[List[float], List[int], List[str]] + + +class SensitivityType(str, Enum): + MONTECARLO = "montecarlo" + SCENARIO = "scenario" + + +class EnsembleSensitivityCase(BaseModel): + """Description/data for a single sensitivity case in an ensemble""" + + name: str + realizations: List[int] + + +class EnsembleSensitivity(BaseModel): + """Description/data for a single sensitivity in an ensemble""" + + name: str + type: SensitivityType + cases: List[EnsembleSensitivityCase] + + +class EnsembleParameters(BaseModel): # Find a better name + """Description/data for all parameters in an ensemble + type: "sensitivity" | "historymatch" | "prediction ??" + """ + + parameters: List[EnsembleParameter] + sensitivities: Optional[List[EnsembleSensitivity]] = None diff --git a/backend_py/primary/tests/integration/conftest.py b/backend_py/primary/tests/integration/conftest.py index 73041ca45..53e04f9d1 100644 --- a/backend_py/primary/tests/integration/conftest.py +++ b/backend_py/primary/tests/integration/conftest.py @@ -21,8 +21,8 @@ class SumoTestEnsemble: ensemble_name: str -@pytest.fixture(name="sumo_test_ensemble_prod", scope="session") -def fixture_sumo_test_ensemble_prod() -> SumoTestEnsemble: +@pytest.fixture(name="sumo_test_ensemble_ahm", scope="session") +def fixture_sumo_test_ensemble_ahm() -> SumoTestEnsemble: return SumoTestEnsemble( field_identifier="DROGON", case_name="webviz_ahm_case", @@ -30,7 +30,14 @@ def fixture_sumo_test_ensemble_prod() -> SumoTestEnsemble: ensemble_name="iter-0", ) - +@pytest.fixture(name="sumo_test_ensemble_design", scope="session") +def fixture_sumo_test_ensemble_design() -> SumoTestEnsemble: + return SumoTestEnsemble( + field_identifier="DROGON", + case_name="01_drogon_design", + case_uuid="b89873c8-6f4d-40e5-978c-afc47beb2a26", + ensemble_name="iter-0", + ) @pytest.fixture(name="test_user", scope="session") def fixture_test_user(): token = "DUMMY_TOKEN_FOR_TESTING" diff --git a/backend_py/primary/tests/integration/routers/explore/test_explore.py b/backend_py/primary/tests/integration/routers/explore/test_explore.py index d42caae83..53e1e4625 100644 --- a/backend_py/primary/tests/integration/routers/explore/test_explore.py +++ b/backend_py/primary/tests/integration/routers/explore/test_explore.py @@ -2,30 +2,31 @@ from primary.routers.explore import schemas -async def test_get_fields(test_user, sumo_test_ensemble_prod) -> None: +async def test_get_fields(test_user, sumo_test_ensemble_ahm) -> None: fields = await router.get_fields(test_user) assert all(isinstance(f, schemas.FieldInfo) for f in fields) - assert any(f.field_identifier == sumo_test_ensemble_prod.field_identifier for f in fields) - -async def test_get_cases(test_user, sumo_test_ensemble_prod) -> None: - cases = await router.get_cases(test_user, sumo_test_ensemble_prod.field_identifier) + assert any(f.field_identifier == sumo_test_ensemble_ahm.field_identifier for f in fields) + + +async def test_get_cases(test_user, sumo_test_ensemble_ahm) -> None: + cases = await router.get_cases(test_user, sumo_test_ensemble_ahm.field_identifier) assert all(isinstance(c, schemas.CaseInfo) for c in cases) - assert any(c.uuid == sumo_test_ensemble_prod.case_uuid for c in cases) + assert any(c.uuid == sumo_test_ensemble_ahm.case_uuid for c in cases) -async def test_get_ensembles(test_user, sumo_test_ensemble_prod) -> None: - ensembles = await router.get_ensembles(test_user, sumo_test_ensemble_prod.case_uuid) +async def test_get_ensembles(test_user, sumo_test_ensemble_ahm) -> None: + ensembles = await router.get_ensembles(test_user, sumo_test_ensemble_ahm.case_uuid) assert all(isinstance(e, schemas.EnsembleInfo) for e in ensembles) - assert any(e.name == sumo_test_ensemble_prod.ensemble_name for e in ensembles) + assert any(e.name == sumo_test_ensemble_ahm.ensemble_name for e in ensembles) -async def test_get_ensemble_details(test_user, sumo_test_ensemble_prod) -> None: +async def test_get_ensemble_details(test_user, sumo_test_ensemble_ahm) -> None: ensemble_details = await router.get_ensemble_details( - test_user, sumo_test_ensemble_prod.case_uuid, sumo_test_ensemble_prod.ensemble_name + test_user, sumo_test_ensemble_ahm.case_uuid, sumo_test_ensemble_ahm.ensemble_name ) assert isinstance(ensemble_details, schemas.EnsembleDetails) - assert ensemble_details.name == sumo_test_ensemble_prod.ensemble_name - assert ensemble_details.field_identifier == sumo_test_ensemble_prod.field_identifier - assert ensemble_details.case_uuid == sumo_test_ensemble_prod.case_uuid - assert ensemble_details.case_name == sumo_test_ensemble_prod.case_name + assert ensemble_details.name == sumo_test_ensemble_ahm.ensemble_name + assert ensemble_details.field_identifier == sumo_test_ensemble_ahm.field_identifier + assert ensemble_details.case_uuid == sumo_test_ensemble_ahm.case_uuid + assert ensemble_details.case_name == sumo_test_ensemble_ahm.case_name assert len(ensemble_details.realizations) == 100 diff --git a/backend_py/primary/tests/integration/routers/parameters/test_parameters.py b/backend_py/primary/tests/integration/routers/parameters/test_parameters.py new file mode 100644 index 000000000..d344c9854 --- /dev/null +++ b/backend_py/primary/tests/integration/routers/parameters/test_parameters.py @@ -0,0 +1,88 @@ +import pytest +import numpy as np + +from primary.routers.parameters import router +from primary.routers.parameters import schemas + + +@pytest.mark.parametrize( + ["exclude_constants", "expected_length", "expected_first_name"], + [ + (True, 74, "FAULT_SEAL_SCALING"), + (False, 113, "DCONV_ALTERNATIVE"), + ], +) +async def test_get_parameter_names_and_description( + test_user, sumo_test_ensemble_ahm, exclude_constants, expected_length, expected_first_name +) -> None: + + parameters = await router.get_parameter_names_and_description( + test_user, + sumo_test_ensemble_ahm.case_uuid, + sumo_test_ensemble_ahm.ensemble_name, + exclude_all_values_constant=exclude_constants, + sort_order="alphabetically", + ) + + assert all(isinstance(p, schemas.EnsembleParameterDescription) for p in parameters) + assert len(parameters) == expected_length + + assert parameters[0].name == expected_first_name + + +@pytest.mark.parametrize( + ["parameter_name", "parameter_group_name", "is_log", "real_count", "expected_mean"], + [ + ("FAULT_SEAL_SCALING", "LOG10_GLOBVAR", True, 100, 2.8239), + ("DCONV_ALTERNATIVE", "GLOBVAR", False, 100, 2.0), + ], +) +async def test_get_parameter( + test_user, sumo_test_ensemble_ahm, parameter_name, parameter_group_name, is_log, real_count, expected_mean +) -> None: + + parameter = await router.get_parameter( + test_user, sumo_test_ensemble_ahm.case_uuid, sumo_test_ensemble_ahm.ensemble_name, parameter_name + ) + + assert isinstance(parameter, schemas.EnsembleParameter) + assert parameter.name == parameter_name + assert parameter.group_name == parameter_group_name + assert parameter.is_logarithmic == is_log + assert len(parameter.realizations) == real_count + assert np.isclose(np.mean(parameter.values), expected_mean, atol=1e-5) + +async def test_is_sensitivity_run_ahm(test_user, sumo_test_ensemble_ahm) -> None: + assert await router.is_sensitivity_run(test_user, sumo_test_ensemble_ahm.case_uuid, sumo_test_ensemble_ahm.ensemble_name) == True + +async def test_is_sensitivity_run_design(test_user, sumo_test_ensemble_design) -> None: + assert await router.is_sensitivity_run(test_user, sumo_test_ensemble_design.case_uuid, sumo_test_ensemble_design.ensemble_name) == True + +async def test_get_sensitivities_ahm(test_user, sumo_test_ensemble_ahm) -> None: + sensitivities = await router.get_sensitivities(test_user, sumo_test_ensemble_ahm.case_uuid, sumo_test_ensemble_ahm.ensemble_name) + assert all(isinstance(s, schemas.EnsembleSensitivity) for s in sensitivities) + assert len(sensitivities) == 1 + sens_case = sensitivities[0] + assert sens_case.name == "APS_input" + assert sens_case.type == schemas.SensitivityType.MONTECARLO + assert len(sens_case.cases) == 1 + sens_case_case = sens_case.cases[0] + assert sens_case_case.name == "p10_p90" + +async def test_get_sensitivities_design(test_user, sumo_test_ensemble_design) -> None: + sensitivities = await router.get_sensitivities(test_user, sumo_test_ensemble_design.case_uuid, sumo_test_ensemble_design.ensemble_name) + assert all(isinstance(s, schemas.EnsembleSensitivity) for s in sensitivities) + assert len(sensitivities) == 11 + sens_case = next(s for s in sensitivities if s.name == "faultseal") + assert sens_case.name == "faultseal" + assert sens_case.type == schemas.SensitivityType.SCENARIO + assert len(sens_case.cases) == 2 + sens_name = next(c for c in sens_case.cases if c.name == "high") + assert set(sens_name.realizations) == set(range(110,120)) + + sens_case = next(s for s in sensitivities if s.name == "hum") + assert sens_case.name == "hum" + assert sens_case.type == schemas.SensitivityType.MONTECARLO + assert len(sens_case.cases) == 1 + sens_name = next(c for c in sens_case.cases if c.name == "p10_p90") + assert set(sens_name.realizations) == set(range(10,20)) diff --git a/backend_py/primary/tests/integration/routers/timeseries/test_get_realizations_vector_data.py b/backend_py/primary/tests/integration/routers/timeseries/test_get_realizations_vector_data.py index 1a9089ab0..4f7796f1b 100644 --- a/backend_py/primary/tests/integration/routers/timeseries/test_get_realizations_vector_data.py +++ b/backend_py/primary/tests/integration/routers/timeseries/test_get_realizations_vector_data.py @@ -15,14 +15,14 @@ ], ) async def test_get_realizations_vector_data_dates( - test_user, sumo_test_ensemble_prod, frequency, date_count, expected_mean + test_user, sumo_test_ensemble_ahm, frequency, date_count, expected_mean ) -> None: realization_data = await router.get_realizations_vector_data( None, test_user, - sumo_test_ensemble_prod.case_uuid, - sumo_test_ensemble_prod.ensemble_name, + sumo_test_ensemble_ahm.case_uuid, + sumo_test_ensemble_ahm.ensemble_name, "FOPT", frequency, ) @@ -43,14 +43,14 @@ async def test_get_realizations_vector_data_dates( ], ) async def test_get_realizations_vector_data_realizations( - test_user, sumo_test_ensemble_prod, realizations, real_count, expected_mean + test_user, sumo_test_ensemble_ahm, realizations, real_count, expected_mean ) -> None: realization_data = await router.get_realizations_vector_data( None, test_user, - sumo_test_ensemble_prod.case_uuid, - sumo_test_ensemble_prod.ensemble_name, + sumo_test_ensemble_ahm.case_uuid, + sumo_test_ensemble_ahm.ensemble_name, "FOPT", schemas.Frequency.YEARLY, realizations, diff --git a/backend_py/primary/tests/integration/routers/timeseries/test_get_vector_list.py b/backend_py/primary/tests/integration/routers/timeseries/test_get_vector_list.py index 14f1b1e61..b42110206 100644 --- a/backend_py/primary/tests/integration/routers/timeseries/test_get_vector_list.py +++ b/backend_py/primary/tests/integration/routers/timeseries/test_get_vector_list.py @@ -2,10 +2,10 @@ from primary.routers.timeseries import schemas -async def test_get_vector_list(test_user, sumo_test_ensemble_prod) -> None: +async def test_get_vector_list(test_user, sumo_test_ensemble_ahm) -> None: vector_list = await router.get_vector_list( - None, test_user, sumo_test_ensemble_prod.case_uuid, sumo_test_ensemble_prod.ensemble_name + None, test_user, sumo_test_ensemble_ahm.case_uuid, sumo_test_ensemble_ahm.ensemble_name ) assert len(vector_list) == 786