updates to data model to reflect more organized structure of output data

biosimulators · Oct 22, 2024 · 8b6d283 · 8b6d283
1 parent 5158a0a
commit 8b6d283
Show file tree

Hide file tree

Showing 5 changed files with 43 additions and 34 deletions.
diff --git a/assets/scripts/build_all.sh b/assets/scripts/build_all.sh
@@ -2,21 +2,18 @@
 
 set -e
 
-version="$1"
-arg1="$2"
-un=AlexPatrie
+arg1="$1"
 
 # optionally prune/clear system and cache prior to build
 if [ "$arg1" == "-p" ]; then
   yes | docker system prune -a
+  yes | docker buildx prune -a
 fi
 
+# remove pycache to clean images
+sudo rm -r compose_api/__pycache__
+sudo rm -r compose_worker/__pycache__
+
 # build and push containers
-./assets/scripts/build_api.sh "$version" "$un" && ./assets/scripts/build_worker.sh "$version" "$un"
+docker compose build --no-cache
 
-# optionally apply changes to container versions in Kustomization
-if [ "$arg1" == "-k" ]; then
-  cd kustomize \
-    && kubectl kustomize overlays/biochecknet | kubectl apply -f - \
-    && cd ..
-fi
diff --git a/compose_api/data_model.py b/compose_api/data_model.py
@@ -68,10 +68,15 @@ class SmoldynJob(IncompleteJob):
 
 # TODO: parse results and make object specs
 class ObservableData(BaseModel):
-    name: str
+    observable_name: str
     mse: Dict[str, Any]   # Dict[str, float]]
     proximity: Dict[str, Any]   #  Dict[str, bool]]
-    output_data: Dict[str, Any]   #  List[float]]
+    output_data: Dict[str, Union[List[float], str]]   #  List[float]]
+
+
+class SimulatorRMSE(BaseModel):
+    simulator: str
+    rmse_matrix: Dict[str, float]
 
 
 class Output(BaseModel):
@@ -97,8 +102,8 @@ class VerificationOutput(Output):
         requested_simulators: List[str]
         results: Optional[dict] = None TODO: parse this
     """
-    requested_simulators: Optional[List[str] ]= None
-    results: Optional[Union[Dict[str, Any], List[ObservableData]]] = None
+    requested_simulators: Optional[List[str]] = None
+    results: Optional[Union[List[Union[ObservableData, SimulatorRMSE, Any]], Dict[str, Any]]] = None
 
 
 class OutputData(BaseModel):

diff --git a/compose_api/main.py b/compose_api/main.py
@@ -19,6 +19,7 @@
 from compose_api.data_model import ObservableData
 # from bio_check import MONGO_URI
 from data_model import (
+    SimulatorRMSE,
     SmoldynJob,
     VerificationOutput,
     SmoldynOutput,
@@ -887,25 +888,25 @@ async def get_verification_output(job_id: str) -> VerificationOutput:
 
     if job is not None:
         job.pop('_id', None)
-        # data = [
-        #     ObservableData(name=obs_name, mse=obs_data['mse'], proximity=obs_data['proximity'], output_data=obs_data['output_data'])
-        #     for obs_name, obs_data in job.get('results').items() if "rmse" not in obs_name
-        # ]
-        # return VerificationOutput(
-        #     job_id=job_id,
-        #     timestamp=job.get('timestamp'),
-        #     status=job.get('status'),
-        #     source=job.get('source'),
-        #     requested_simulators=job.get('requested_simulators'),
-        #     results=data
-        # )
+        results = job.get('results')
+        data = []
+        if results is not None:
+            for name, obs_data in results.items():
+                if not name == "rmse":
+                    obs = ObservableData(observable_name=name, mse=obs_data['mse'], proximity=obs_data['proximity'], output_data=obs_data['output_data'])
+                    data.append(obs)
+                else:
+                    for simulator_name, data_table in obs_data.items():
+                        obs = SimulatorRMSE(simulator=simulator_name, rmse_matrix=data_table)
+                        data.append(obs)
 
         output = VerificationOutput(
             job_id=job_id,
             timestamp=job.get('timestamp'),
             status=job.get('status'),
             source=job.get('source', job.get('path')).split('/')[-1],
-            results=job.get('results')
+            results=data,  # Change this to the results below if there is an issue
+            # results=job.get('results')
         )
         requested_simulators = job.get('simulators', job.get('requested_simulators'))
         if requested_simulators is not None:

diff --git a/compose_worker/supervisor.py b/compose_worker/supervisor.py
@@ -132,13 +132,19 @@ async def _check(self):
                 # change job status for client by inserting a new in progress job
                 job_in_progress = self.job_exists(job_id=job_id, collection_name="in_progress_jobs")
                 if not job_in_progress:
-                    in_progress_entry = {'job_id': job_id, 'timestamp': self.db_connector.timestamp(), 'status': JobStatus.IN_PROGRESS.value, 'source': source}
+                    in_progress_entry = {
+                        'job_id': job_id,
+                        'timestamp': self.db_connector.timestamp(),
+                        'status': JobStatus.IN_PROGRESS.value,
+                        'requested_simulators': pending_job.get('simulators'),
+                        'source': source
+                    }
 
                     # special handling of composition jobs TODO: move this to the supervisor below
-                    if job_id.startswith('composition-run'):
-                        in_progress_entry['composite_spec'] = pending_job.get('composite_spec')
-                        in_progress_entry['simulator'] = pending_job.get('simulators')
-                        in_progress_entry['duration'] = pending_job.get('duration')
+                    # if job_id.startswith('composition-run'):
+                    #     in_progress_entry['composite_spec'] = pending_job.get('composite_spec')
+                    #     in_progress_entry['simulator'] = pending_job.get('simulators')
+                    #     in_progress_entry['duration'] = pending_job.get('duration')
 
                     in_progress_job = await self.db_connector.insert_job_async(
                         collection_name="in_progress_jobs",

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -15,7 +15,7 @@ services:
     build:
       context: ./compose_api
       dockerfile: ./Dockerfile-compose_api
-    image: ghcr.io/biosimulators/bio-check-compose_api:1.11.8  # .3
+    image: ghcr.io/biosimulators/bio-check-compose_api:1.11.9  # .8
     container_name: api
     environment:
       - MONGO_URI=mongodb://mongodb:27017/?retryWrites=true&w=majority&appName=bio-compose
@@ -34,7 +34,7 @@ services:
       context: ./compose_worker
       dockerfile: ./Dockerfile-compose_worker
 
-    image: ghcr.io/biosimulators/bio-check-compose_worker:1.11.6  # .5
+    image: ghcr.io/biosimulators/bio-check-compose_worker:1.11.7  # .6
     container_name: worker
     environment:
       - MONGO_URI=mongodb://mongodb:27017/?retryWrites=true&w=majority&appName=bio-compose