Merge remote-tracking branch 'origin/master' into 1154-biomodels-endp…

…oint

python-utils/vcutils/vcell_pipeline/prune_already_published_models.py

@@ -0,0 +1,28 @@
+import os
+import re
+from pathlib import Path
+from typing import Optional
+
+import requests
+from pydantic import BaseModel
+
+from vcutils.common.api_utils import download_file
+from vcutils.vcell_pipeline.citation import getCitation, CitationInfo, getSuggestedProjectName
+from vcutils.vcell_pipeline.datamodels import Publication
+
+def prune_already_published_models(target_dir: Path):
+    response = requests.get("https://api.biosimulations.org/projects", headers={'accept': 'application/json'})
+    publications: list[dict] = response.json()
+    ids = [publication["id"] for publication in publications]
+    vcdb_pubs: set[str] = set([elem for elem in ids if "VCDB" in elem])
+    vcdb_ids = set([re.split("[\-_]+", parts)[1] for parts in vcdb_pubs])
+    files = {re.split("[\-_]+", file)[1] : file for file in os.listdir(target_dir)}
+
+    for potential_id in vcdb_ids:
+        if potential_id in files:
+            os.remove(os.path.join(target_dir, files[potential_id]))
+            print(f"Removed file: {files[potential_id]}")
+
+
+if __name__ == "__main__":
+    prune_already_published_models("/home/ldrescher/Development/BioSim/publication_candidates/input")

python-utils/vcutils/vcell_pipeline/rename_vcell_omex.py

@@ -43,7 +43,10 @@ def rename_published_omex(api_base_url: str, subject_dir: Path) -> None:
             continue
         if file_key in id_to_name_mapping:
             changes_count += 1
-            os.rename(os.path.join(subject_dir, file), os.path.join(subject_dir, id_to_name_mapping[file_key]))
+            new_name = id_to_name_mapping[file_key]
+            if new_name[-5:] != ".omex":
+                new_name += ".omex"
+            os.rename(os.path.join(subject_dir, file), os.path.join(subject_dir, new_name))
         else:
             print(" > No corresponding file to rename with key: {}".format(file_key))
 
@@ -56,5 +59,5 @@ def rename_published_omex(api_base_url: str, subject_dir: Path) -> None:
 if __name__ == "__main__":
     rename_published_omex(
         api_base_url="https://vcellapi-beta.cam.uchc.edu:8080/api/v0",
-        subject_dir=Path("/home/ldrescher/Documents/convertedFiles")
+        subject_dir=Path("/home/ldrescher/VCell/vcdb/pretty_name_exchange/")
     )

vcell-cli/src/main/java/org/vcell/cli/run/SedmlJob.java

@@ -389,7 +389,7 @@ private void generateHDF5(SolverHandler solverHandler, HDF5ExecutionResults mast
 
         Hdf5DataExtractor hdf5Extractor = new Hdf5DataExtractor(this.sedml, solverHandler.taskToTempSimulationMap, this.RESULTS_DIRECTORY_PATH);
 
-        Hdf5DataContainer partialHdf5File = hdf5Extractor.extractHdf5RelevantData(solverHandler.nonSpatialResults, solverHandler.spatialResults);
+        Hdf5DataContainer partialHdf5File = hdf5Extractor.extractHdf5RelevantData(solverHandler.nonSpatialResults, solverHandler.spatialResults, masterHdf5File.isBioSimHdf5);
         masterHdf5File.addResults(this.sedml, partialHdf5File);
 
         for (File tempH5File : solverHandler.spatialResults.values()) {

vcell-cli/src/main/java/org/vcell/cli/run/hdf5/Hdf5DataExtractor.java

@@ -50,9 +50,9 @@ public Hdf5DataExtractor(SedML sedml, Map<AbstractTask, TempSimulation> taskToSi
      * @see NonspatialResultsConverter::convertNonspatialResultsToSedmlFormat
      * @see SpatialResultsConverter::collectSpatialDatasets
      */
-    public Hdf5DataContainer extractHdf5RelevantData(Map<TaskJob, SBMLNonspatialSimResults> nonSpatialResults, Map<TaskJob, File> spatialResults) {
+    public Hdf5DataContainer extractHdf5RelevantData(Map<TaskJob, SBMLNonspatialSimResults> nonSpatialResults, Map<TaskJob, File> spatialResults, boolean isBioSimMode) {
         Map<Report, List<Hdf5SedmlResults>> wrappers = new LinkedHashMap<>();
-        Hdf5DataContainer hdf5FileWrapper = new Hdf5DataContainer();
+        Hdf5DataContainer hdf5FileWrapper = new Hdf5DataContainer(isBioSimMode);
         Exception nonSpatialException = null, spatialException = null;
 
         try {

vcell-cli/src/main/java/org/vcell/cli/run/hdf5/Hdf5DataPreparer.java

@@ -124,7 +124,7 @@ public static Hdf5PreparedData prepareNonspatialData(Hdf5SedmlResults datasetWra
         List<Long> dataDimensionList = new LinkedList<>();
 
         Hdf5SedmlResultsNonspatial dataSourceNonspatial = (Hdf5SedmlResultsNonspatial) datasetWrapper.dataSource;
-        Map<DataSet, List<double[]>> dataSetMap = dataSourceNonspatial.allJobResults;
+        Map<DataSet, List<double[]>> dataSetMap = dataSourceNonspatial.dataItems.getDataSetMappings(report);
         numDataSets = dataSetMap.keySet().size();
 
         for (DataSet dataSet : dataSetMap.keySet()){
@@ -150,7 +150,7 @@ public static Hdf5PreparedData prepareNonspatialData(Hdf5SedmlResults datasetWra
         int bufferOffset = 0;
 
         for (DataSet dataSet : dataSetMap.keySet()) {
-            List<double[]> dataArrays = dataSourceNonspatial.allJobResults.get(dataSet);
+            List<double[]> dataArrays = dataSourceNonspatial.dataItems.get(report, dataSet);
             for (double[] dataArray : dataArrays){
                 if (dataArray.length < numTimePoints){
                     double[] paddedArray = new double[Math.toIntExact(numTimePoints)];

vcell-cli/src/main/java/org/vcell/cli/run/hdf5/Hdf5SedmlResultsNonspatial.java

@@ -14,6 +14,6 @@ public Hdf5SedmlResultsNonspatial() {}
     /**
      * Map of all data contained within a job relevant to HDF5 formatted files
      */
-    public Map<DataSet, List<double[]>> allJobResults = new LinkedHashMap<>();
+    public NonspatialDataMapping dataItems = new NonspatialDataMapping();
 
 }

vcell-cli/src/main/java/org/vcell/cli/run/hdf5/NonspatialDataMapping.java

@@ -0,0 +1,45 @@
+package org.vcell.cli.run.hdf5;
+
+import org.jlibsedml.DataSet;
+import org.jlibsedml.Report;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+public class NonspatialDataMapping {
+    private final Map<Report, Map<DataSet, List<double[]>>> dataMapping;
+
+    public NonspatialDataMapping(){
+        this.dataMapping = new LinkedHashMap<>();
+    }
+
+    public List<double[]> put(Report report, DataSet dataset, List<double[]> data){
+        if (report == null || dataset == null) throw new IllegalArgumentException("No null values allowed!");
+        Map<DataSet, List<double[]>> reportMap;
+        if (!this.dataMapping.containsKey(report) ) {
+            this.dataMapping.put(report, new LinkedHashMap<>());
+        }
+        reportMap = this.dataMapping.get(report);
+        if (!reportMap.containsKey(dataset)) return reportMap.put(dataset, data);
+        String errorMessage = String.format("Data already recorded for Report={%s}, DataSet={%s}", report.getName(), dataset.getLabel());
+        throw new IllegalArgumentException(errorMessage);
+    }
+
+    public List<double[]> get(Report report, DataSet dataset){
+        Map<DataSet, List<double[]>> reportMap = this.getDataSetMappings(report);
+        List<double[]> result = reportMap.get(dataset);
+        if (result != null) return result;
+        String errorMessage = String.format("No data can be found for Report={%s}, DataSet={%s}", report.getName(), dataset.getLabel());
+        throw new IllegalArgumentException(errorMessage);
+    }
+
+    public Map<DataSet, List<double[]>> getDataSetMappings(Report report){
+        if (this.dataMapping.containsKey(report)) return this.dataMapping.get(report);
+        throw new IllegalArgumentException(String.format("No data for Report={%s} was found", report.getName()));
+    }
+
+    public boolean isEmpty(){
+        return this.dataMapping.isEmpty();
+    }
+}

vcell-cli/src/main/java/org/vcell/cli/run/hdf5/NonspatialResultsConverter.java

@@ -174,11 +174,11 @@ public static Map<Report, List<Hdf5SedmlResults>> convertNonspatialResultsToSedm
             for (DataSet dataSet : dataSetValues.keySet()){
                 NonspatialValueHolder dataSetValuesSource = dataSetValues.get(dataSet);
 
-                dataSourceNonspatial.allJobResults.put(dataSet, new LinkedList<>());
+                dataSourceNonspatial.dataItems.put(report, dataSet, new LinkedList<>());
                 dataSourceNonspatial.scanBounds = dataSetValuesSource.vcSimulation.getMathOverrides().getScanBounds();
                 dataSourceNonspatial.scanParameterNames = dataSetValuesSource.vcSimulation.getMathOverrides().getScannedConstantNames();
                 for (double[] data : dataSetValuesSource.listOfResultSets) {
-                    dataSourceNonspatial.allJobResults.get(dataSet).add(data);
+                    dataSourceNonspatial.dataItems.get(report, dataSet).add(data);
                     shapes.add(Integer.toString(data.length));
                 }
 

vcell-cli/src/test/java/org/vcell/cli/run/hdf5/Hdf5WriterTest.java

@@ -20,9 +20,14 @@ public class Hdf5WriterTest {
 
     public static HDF5ExecutionResults createExampleData() {
 
+        Report plotReport = new Report("report0", "Plot Report");
+        Report reportReport = new Report("report1", "Report Report");
+
         DataSet t = new DataSet("t","t","t","#null");
         DataSet s0 = new DataSet("s0","s0","s0","#null");
         DataSet s1 = new DataSet("s1", "s1", "s1","#null");
+        plotReport.addDataSet(t); plotReport.addDataSet(s0); plotReport.addDataSet(s1);
+        reportReport.addDataSet(t); reportReport.addDataSet(s0); reportReport.addDataSet(s1);
 
         Hdf5SedmlMetadata plotMetadata = new Hdf5SedmlMetadata();
         plotMetadata._type = "SedPlot2D";
@@ -58,33 +63,31 @@ public static HDF5ExecutionResults createExampleData() {
         plotDataSourceNonspatial.scanBounds = new int[0];
         plotDataSourceNonspatial.scanParameterNames = new String[0];
         plotDatasetWrapper.dataSource = plotDataSourceNonspatial;
-        Map<DataSet, List<double[]>> plotJob = new LinkedHashMap<>();
-        plotJob.put(t, Arrays.asList(row_t));
-        plotJob.put(s0, Arrays.asList(row_S0_0));
-        plotJob.put(s1, Arrays.asList(row_S1));
-        plotDataSourceNonspatial.allJobResults = plotJob;
-
+        plotDataSourceNonspatial.dataItems.put(plotReport, t, Arrays.asList(row_t));
+        plotDataSourceNonspatial.dataItems.put(plotReport, s0, Arrays.asList(row_S0_0));
+        plotDataSourceNonspatial.dataItems.put(plotReport, s1, Arrays.asList(row_S1));
 
         Hdf5SedmlResults reportDatasetWrapper = new Hdf5SedmlResults();
         reportDatasetWrapper.datasetMetadata = reportMetadata;
         Hdf5SedmlResultsNonspatial reportDataSourceNonspatial = new Hdf5SedmlResultsNonspatial();
         reportDataSourceNonspatial.scanBounds = new int[] { 2 }; // zero indexed? 
         reportDataSourceNonspatial.scanParameterNames = new String[] { "k1" };
         reportDatasetWrapper.dataSource = reportDataSourceNonspatial;
-        Map<DataSet, List<double[]>> reportJob = new LinkedHashMap<>();
-        reportJob.put(t, Arrays.asList(row_t, row_t, row_t));
-        reportJob.put(s0, Arrays.asList(row_S0_0, row_S0_1, row_S0_2));
-        reportJob.put(s1, Arrays.asList(row_S1, row_S1, row_S1));
-        reportDataSourceNonspatial.allJobResults = reportJob;
+        reportDataSourceNonspatial.dataItems.put(reportReport, t, Arrays.asList(row_t, row_t, row_t));
+        reportDataSourceNonspatial.dataItems.put(reportReport, s0, Arrays.asList(row_S0_0, row_S0_1, row_S0_2));
+        reportDataSourceNonspatial.dataItems.put(reportReport, s1, Arrays.asList(row_S1, row_S1, row_S1));
 
         Hdf5DataContainer hdf5FileWrapper = new Hdf5DataContainer();
-        Report report = new Report("simId","simName");
-        String uri = "___0_export_NO_scan_test.sedml";
-        List<Hdf5SedmlResults> wrappers = new ArrayList<>();
-        wrappers.add(plotDatasetWrapper);
-        wrappers.add(reportDatasetWrapper);
-        hdf5FileWrapper.reportToUriMap.put(report, uri);
-        hdf5FileWrapper.reportToResultsMap.put(report, wrappers);
+        String uri1 = "___0_export_NO_scan_test_1.sedml";
+        String uri2 = "___0_export_NO_scan_test_2.sedml";
+        List<Hdf5SedmlResults> wrappers1 = new ArrayList<>();
+        List<Hdf5SedmlResults> wrappers2 = new ArrayList<>();
+        wrappers1.add(plotDatasetWrapper);
+        wrappers2.add(reportDatasetWrapper);
+        hdf5FileWrapper.reportToUriMap.put(plotReport, uri1);
+        hdf5FileWrapper.reportToUriMap.put(reportReport, uri2);
+        hdf5FileWrapper.reportToResultsMap.put(plotReport, wrappers1);
+        hdf5FileWrapper.reportToResultsMap.put(reportReport, wrappers2);
 
         HDF5ExecutionResults results = new HDF5ExecutionResults(true);
         results.addResults(null, hdf5FileWrapper);