Merge remote-tracking branch 'equinor/main' into adjust-test-setup

SECURITY.md

@@ -0,0 +1,16 @@
+If you discover a security vulnerability in this project, please follow these steps to responsibly disclose it:
+
+1. **Do not** create a public GitHub issue for the vulnerability.
+
+2. Follow our guideline for Responsible Disclosure Policy at https://www.equinor.com/about-us/csirt to report the issue
+
+The following information will help us triage your report more quickly:
+
+-   Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
+-   Full paths of source file(s) related to the manifestation of the issue
+-   The location of the affected source code (tag/branch/commit or direct URL)
+-   Any special configuration required to reproduce the issue
+-   Step-by-step instructions to reproduce the issue
+-   Proof-of-concept or exploit code (if possible)
+-   Impact of the issue, including how an attacker might exploit the issue
+-   We prefer all communications to be in English.

backend/pyproject.toml

@@ -24,7 +24,7 @@ httpx = "^0.24.0"
 psutil = "^5.9.5"
 vtk = "^9.2.6"
 fmu-sumo = "^0.5.4"
-sumo-wrapper-python = "^0.4.0"
+sumo-wrapper-python = "^0.4.1"
 
 
 [tool.poetry.group.dev.dependencies]

backend/src/config.py

@@ -15,7 +15,7 @@
 CLIENT_SECRET = os.environ["WEBVIZ_CLIENT_SECRET"]
 SMDA_SUBSCRIPTION_KEY = os.environ["WEBVIZ_SMDA_SUBSCRIPTION_KEY"]
 SMDA_RESOURCE_SCOPE = os.environ["WEBVIZ_SMDA_RESOURCE_SCOPE"]
-SUMO_ENV = os.getenv("WEBVIZ_SUMO_ENV", "dev")
+SUMO_ENV = os.getenv("WEBVIZ_SUMO_ENV", "prod")
 GRAPH_SCOPES = ["User.Read", "User.ReadBasic.All"]
 
 RESOURCE_SCOPES_DICT = {

backend/src/services/sumo_access/parameter_access.py

@@ -1,10 +1,12 @@
 import logging
+from io import BytesIO
 from typing import List, Optional
 
-import numpy as np
 import pandas as pd
+import pyarrow as pa
+import pyarrow.parquet as pq
 
-from .queries.parameters import get_parameters_for_iteration, SumoEnsembleParameter
+from ..utils.perf_timer import PerfTimer
 from ._helpers import SumoEnsemble
 from .parameter_types import (
     EnsembleParameter,
@@ -20,17 +22,31 @@
 class ParameterAccess(SumoEnsemble):
     async def get_parameters_and_sensitivities(self) -> EnsembleParameters:
         """Retrieve parameters for an ensemble"""
+        timer = PerfTimer()
 
-        # Sumo query. Replace with explorer when ready
-        sumo_parameters: List[SumoEnsembleParameter] = await get_parameters_for_iteration(
-            self._sumo_client, self._case_uuid, self._iteration_name
+        table_collection = self._case.tables.filter(
+            iteration=self._iteration_name,
+            aggregation="collection",
+            name="parameters",
+            tagname="all",
         )
+        if await table_collection.length_async() == 0:
+            raise ValueError(f"No parameter tables found {self._case.name, self._iteration_name}")
+        if await table_collection.length_async() > 1:
+            raise ValueError(f"Multiple parameter tables found {self._case.name,self._iteration_name}")
 
-        sensitivities = create_ensemble_sensitivities(sumo_parameters)
-        parameters = [create_ensemble_parameter(sumo_parameter) for sumo_parameter in sumo_parameters]
+        table = await table_collection.getitem_async(0)
+        byte_stream: BytesIO = await table.blob_async
+        table = pq.read_table(byte_stream)
+
+        et_download_arrow_table_ms = timer.lap_ms()
+        LOGGER.debug(f"Downloaded arrow table in {et_download_arrow_table_ms}ms")
+
+        ensemble_parameters = parameter_table_to_ensemble_parameters(table)
+        sensitivities = create_ensemble_sensitivities(ensemble_parameters)
 
         return EnsembleParameters(
-            parameters=parameters,
+            parameters=ensemble_parameters,
             sensitivities=sensitivities,
         )
 
@@ -40,27 +56,8 @@ async def get_parameter(self, parameter_name: str) -> EnsembleParameter:
         return next(parameter for parameter in parameters.parameters if parameter.name == parameter_name)
 
 
-def create_ensemble_parameter(
-    sumo_parameter: SumoEnsembleParameter,
-) -> EnsembleParameter:
-    """Create an EnsembleParameter from a Sumo parameter object"""
-
-    return EnsembleParameter(
-        name=sumo_parameter.name,
-        is_logarithmic=sumo_parameter.name.startswith("LOG10_"),
-        is_numerical=is_array_numeric(np.array(sumo_parameter.values)),
-        is_constant=all(value == sumo_parameter.values[0] for value in sumo_parameter.values),
-        group_name=sumo_parameter.groupname,
-        descriptive_name=f"{sumo_parameter.name} (log)"
-        if sumo_parameter.name.startswith("LOG10_")
-        else sumo_parameter.name,
-        values=sumo_parameter.values,
-        realizations=sumo_parameter.realizations,
-    )
-
-
 def create_ensemble_sensitivities(
-    sumo_ensemble_parameters: List[SumoEnsembleParameter],
+    sumo_ensemble_parameters: List[EnsembleParameter],
 ) -> Optional[List[EnsembleSensitivity]]:
     """Extract sensitivities from a list of SumoEnsembleParameter objects"""
     sensitivities = []
@@ -115,6 +112,31 @@ def create_ensemble_sensitivity_cases(
     return cases
 
 
-def is_array_numeric(array: np.ndarray) -> bool:
-    """Check if an array is numeric"""
-    return np.issubdtype(array.dtype, np.number)
+def parameter_table_to_ensemble_parameters(parameter_table: pa.Table) -> List[EnsembleParameter]:
+    """Convert a parameter table to an EnsembleParameter"""
+    ensemble_parameters: List[EnsembleParameter] = []
+    for column_name in parameter_table.column_names:
+        if column_name == "REAL":
+            continue
+        parameter_name_components = column_name.split(":")
+        if len(parameter_name_components) > 2:
+            raise ValueError(f"Parameter {column_name} has too many componenents. Expected <groupname>:<parametername>")
+        if len(parameter_name_components) == 1:
+            parameter_name = column_name
+            group_name = None
+        else:
+            group_name = parameter_name_components[0]
+            parameter_name = parameter_name_components[1]
+        ensemble_parameters.append(
+            EnsembleParameter(
+                name=parameter_name,
+                is_logarithmic=column_name.startswith("LOG10_"),
+                is_numerical=parameter_table.schema.field(column_name).type != pa.string,
+                is_constant=len(set(parameter_table[column_name])) == 1,
+                group_name=group_name,
+                descriptive_name=parameter_name,
+                values=parameter_table[column_name].to_numpy().tolist(),
+                realizations=parameter_table["REAL"].to_numpy().tolist(),
+            )
+        )
+    return ensemble_parameters

backend/src/services/sumo_access/summary_access.py

@@ -6,7 +6,7 @@
 import pyarrow as pa
 import pyarrow.compute as pc
 import pyarrow.parquet as pq
-from fmu.sumo.explorer.objects import Case
+from fmu.sumo.explorer.objects import Case, TableCollection
 
 from src.services.utils.arrow_helpers import sort_table_on_real_then_date
 from src.services.utils.perf_timer import PerfTimer
@@ -24,13 +24,7 @@ class SummaryAccess(SumoEnsemble):
     async def get_available_vectors(self) -> List[VectorInfo]:
         timer = PerfTimer()
 
-        smry_table_collection = self._case.tables.filter(
-            aggregation="collection",
-            name="summary",
-            tagname="eclipse",
-            iteration=self._iteration_name,
-        )
-
+        smry_table_collection = await get_smry_table_collection(self._case, self._iteration_name)
         column_names = await smry_table_collection.columns_async
 
         ret_info_arr: List[VectorInfo] = []
@@ -228,19 +222,13 @@ async def get_timestamps(
 async def _load_arrow_table_for_from_sumo(case: Case, iteration_name: str, vector_name: str) -> Optional[pa.Table]:
     timer = PerfTimer()
 
-    smry_table_collection = case.tables.filter(
-        aggregation="collection",
-        name="summary",
-        tagname="eclipse",
-        iteration=iteration_name,
-        column=vector_name,
-    )
+    smry_table_collection = await get_smry_table_collection(case, iteration_name, column_name=vector_name)
     if await smry_table_collection.length_async() == 0:
         return None
     if await smry_table_collection.length_async() > 1:
         raise ValueError(f"Multiple tables found for vector {vector_name=}")
 
-    sumo_table = smry_table_collection[0]
+    sumo_table = await smry_table_collection.getitem_async(0)
     # print(f"{sumo_table.format=}")
     et_locate_sumo_table_ms = timer.lap_ms()
 
@@ -249,7 +237,7 @@ async def _load_arrow_table_for_from_sumo(case: Case, iteration_name: str, vecto
     # The tables we have seen so far have format set to 'arrow', but the actual data is in parquet format.
     # This must be a bug or a misunderstanding.
     # For now, just read the parquet data into an arrow table
-    byte_stream: BytesIO = sumo_table.blob
+    byte_stream: BytesIO = await sumo_table.blob_async
     table = pq.read_table(byte_stream)
     et_download_arrow_table_ms = timer.lap_ms()
 
@@ -302,3 +290,22 @@ def _construct_historical_vector_name(non_historical_vector_name: str) -> Option
         return hist_vec
 
     return None
+
+
+async def get_smry_table_collection(
+    case: Case, iteration_name: str, column_name: Optional[str] = None
+) -> TableCollection:
+    """Get a collection of summary tables for a case and iteration"""
+    all_smry_table_collections = case.tables.filter(
+        aggregation="collection",
+        tagname="summary",
+        iteration=iteration_name,
+        column=column_name,
+    )
+    table_names = await all_smry_table_collections.names_async
+    if len(table_names) == 0:
+        raise ValueError("No summary table collections found")
+    if len(table_names) == 1:
+        return all_smry_table_collections
+
+    raise ValueError(f"Multiple summary table collections found: {table_names}. Expected only one.")