Merge pull request #1217 from virtualcell/PublicationFixes

Fix remaining issues preventing / hindering publishing VCell models on BSTS

python-utils/vcutils/vcell_pipeline/download_vcell_omex.py

@@ -76,7 +76,7 @@ def download_published_omex(api_base_url: str, out_dir: Path) -> None:
 
 if __name__ == "__main__":
     download_published_omex(
-        # api_base_url="https://vcellapi-beta.cam.uchc.edu:8080",
-        api_base_url="https://localhost:8083",
+        api_base_url="https://vcellapi-beta.cam.uchc.edu:8080",
+        # api_base_url="https://localhost:8083",
         out_dir=Path("/Users/schaff/Documents/workspace/vcdb/published/biomodel/omex/sbml")
     )

python-utils/vcutils/vcell_pipeline/rename_vcell_omex.py

@@ -0,0 +1,60 @@
+import os
+from pathlib import Path
+from typing import Optional
+from download_vcell_omex import ExportStatus
+
+import requests
+from pydantic import BaseModel
+
+from vcutils.common.api_utils import download_file
+from vcutils.vcell_pipeline.citation import getCitation, CitationInfo, getSuggestedProjectName
+from vcutils.vcell_pipeline.datamodels import Publication
+
+VERIFY_SSL = False
+
+
+def rename_published_omex(api_base_url: str, subject_dir: Path) -> None:
+    response = requests.get(f"{api_base_url}/publication", headers={'Accept': 'application/json'}, verify=VERIFY_SSL)
+    publication_json = response.json()
+    id_to_name_mapping: dict[str, str] = {}
+    files_to_check = os.listdir(subject_dir)
+    starting_count = len(files_to_check)
+    changes_count = 0
+
+    for pub in [Publication(**jsonDict) for jsonDict in publication_json]:
+        if len(pub.biomodelReferences) == 0:
+            continue
+        print(f"Processing {pub.pubKey}, title: {pub.title}, year: {pub.year}, bimodels: {pub.biomodelReferences}")
+        bmKey = pub.biomodelReferences[0].bmKey
+        pubmedId: Optional[str] = pub.pubmedid
+        citationInfo: Optional[CitationInfo] = None
+        try:
+            citationInfo = getCitation(pubmedId)
+        except Exception as e:
+            print(f"Error getting citation for {pubmedId}: {e}")
+
+        for bioModelKey in [bmr.bmKey for bmr in pub.biomodelReferences]:
+            suggested_project_name = getSuggestedProjectName(bm_key=bmKey, pub_info=pub, citation_info=citationInfo)
+            id_to_name_mapping[bioModelKey] = suggested_project_name
+
+    for file in files_to_check:
+        file_key = file[9:-5]
+        if not file_key.isdigit():
+            continue
+        if file_key in id_to_name_mapping:
+            changes_count += 1
+            os.rename(os.path.join(subject_dir, file), os.path.join(subject_dir, id_to_name_mapping[file_key]))
+        else:
+            print(" > No corresponding file to rename with key: {}".format(file_key))
+
+    ending_count = len(os.listdir(subject_dir))
+    print(f"\t> Starting files: {starting_count}\n\t> Changes Made: {changes_count}\n\t> Finished files: {ending_count}")
+
+
+
+
+if __name__ == "__main__":
+    rename_published_omex(
+        api_base_url="https://vcellapi-beta.cam.uchc.edu:8080",
+        subject_dir=Path("/home/ldrescher/Documents/convertedFiles")
+    )

vcell-cli/src/main/java/org/vcell/cli/run/ExecutionJob.java

@@ -3,6 +3,7 @@
 import org.vcell.cli.CLIRecordable;
 import org.vcell.cli.PythonStreamException;
 import org.vcell.cli.exceptions.ExecutionException;
+import org.vcell.cli.run.hdf5.HDF5ExecutionResults;
 import org.vcell.cli.run.hdf5.Hdf5DataContainer;
 import org.vcell.util.FileUtils;
 
@@ -124,7 +125,7 @@ public void preprocessArchive() throws PythonStreamException, IOException {
      */
     public void executeArchive(boolean isBioSimSedml) throws HDF5Exception, PythonStreamException, ExecutionException {
         try {
-            Hdf5DataContainer masterHdf5File = new Hdf5DataContainer(isBioSimSedml);
+            HDF5ExecutionResults masterHdf5File = new HDF5ExecutionResults(isBioSimSedml);
             this.queueAllSedml();
 
             for (String sedmlLocation : this.sedmlLocations){

vcell-cli/src/main/java/org/vcell/cli/run/SedmlJob.java

@@ -10,6 +10,7 @@
 import org.vcell.cli.CLIRecordable;
 import org.vcell.cli.PythonStreamException;
 import org.vcell.cli.exceptions.ExecutionException;
+import org.vcell.cli.run.hdf5.HDF5ExecutionResults;
 import org.vcell.cli.run.hdf5.Hdf5DataContainer;
 import org.vcell.cli.run.hdf5.Hdf5DataExtractor;
 import org.vcell.cli.trace.Span;
@@ -212,7 +213,7 @@ public boolean preProcessDoc() throws PythonStreamException, InterruptedExceptio
      * @throws PythonStreamException if calls to the python-shell instance are not working correctly
      * @throws IOException if there are system I/O issues
      */
-    public boolean simulateSedml(Hdf5DataContainer masterHdf5File) throws InterruptedException, PythonStreamException, IOException {
+    public boolean simulateSedml(HDF5ExecutionResults masterHdf5File) throws InterruptedException, PythonStreamException, IOException {
         /*  temp code to test plot name correctness
         String idNamePlotsMap = utils.generateIdNamePlotsMap(sedml, outDirForCurrentSedml);
         utils.execPlotOutputSedDoc(inputFile, idNamePlotsMap, this.resultsDirPath);
@@ -280,7 +281,7 @@ private void runSimulations(SolverHandler solverHandler, ExternalDocInfo externa
         this.recordRunDetails(solverHandler);
     }
 
-    private void processOutputs(SolverHandler solverHandler, Hdf5DataContainer masterHdf5File) throws InterruptedException, ExecutionException, PythonStreamException {
+    private void processOutputs(SolverHandler solverHandler, HDF5ExecutionResults masterHdf5File) throws InterruptedException, ExecutionException, PythonStreamException {
         // WARNING!!! Current logic dictates that if any task fails we fail the sedml document
         // change implemented on Nov 11, 2021
         // Previous logic was that if at least one task produces some results we declare the sedml document status as successful
@@ -382,14 +383,14 @@ private void generatePlots() throws PythonStreamException, InterruptedException,
         }
     }
 
-    private void generateHDF5(SolverHandler solverHandler, Hdf5DataContainer masterHdf5File) {
+    private void generateHDF5(SolverHandler solverHandler, HDF5ExecutionResults masterHdf5File) {
         this.logDocumentMessage += "Generating HDF5 file... ";
         logger.info("Generating HDF5 file... ");
 
         Hdf5DataExtractor hdf5Extractor = new Hdf5DataExtractor(this.sedml, solverHandler.taskToTempSimulationMap, this.RESULTS_DIRECTORY_PATH);
 
         Hdf5DataContainer partialHdf5File = hdf5Extractor.extractHdf5RelevantData(solverHandler.nonSpatialResults, solverHandler.spatialResults);
-        masterHdf5File.incorporate(partialHdf5File); // Add the data to the master hdf5 file wrapper.
+        masterHdf5File.addResults(this.sedml, partialHdf5File);
 
         for (File tempH5File : solverHandler.spatialResults.values()) {
             if (tempH5File == null) continue;

vcell-cli/src/main/java/org/vcell/cli/run/hdf5/HDF5ExecutionResults.java

@@ -0,0 +1,35 @@
+package org.vcell.cli.run.hdf5;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.jlibsedml.SedML;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+public class HDF5ExecutionResults implements Iterable<SedML>{
+    private final static Logger logger = LogManager.getLogger(HDF5ExecutionResults.class);
+    private final Map<SedML, Hdf5DataContainer> executionResultsMapping;
+    public boolean isBioSimHdf5;
+
+    public HDF5ExecutionResults(boolean isBioSimHdf5){
+        this.executionResultsMapping = new HashMap<>();
+        this.isBioSimHdf5 = isBioSimHdf5;
+    }
+
+    public void addResults(SedML sedml, Hdf5DataContainer dataContainer){
+        if (this.executionResultsMapping.containsKey(sedml)) logger.warn("Overwriting Results...");
+        this.executionResultsMapping.put(sedml, dataContainer);
+    }
+
+    public Hdf5DataContainer getData(SedML sedml){
+        if (!this.executionResultsMapping.containsKey(sedml)) throw new RuntimeException("No data for requested SED-ML!");
+        return this.executionResultsMapping.get(sedml);
+    }
+
+    @Override
+    public Iterator<SedML> iterator() {
+        return this.executionResultsMapping.keySet().iterator();
+    }
+}

vcell-cli/src/main/java/org/vcell/cli/run/hdf5/Hdf5DataContainer.java

@@ -13,6 +13,7 @@ public class Hdf5DataContainer {
     /**
      *  Uri to results map
      */
+
     public Map<Report, List<Hdf5SedmlResults>> reportToResultsMap;
     public Map<Report, String> reportToUriMap;
     public boolean trackSubSetsInReports;

vcell-cli/src/main/java/org/vcell/cli/run/hdf5/Hdf5DataExtractor.java

@@ -96,6 +96,5 @@ private static void addWrappers(Map<Report, List<Hdf5SedmlResults>> wrappers, Ma
             if (!wrappers.containsKey(potentiallyNewReport)) wrappers.put(potentiallyNewReport, new LinkedList<>());
             wrappers.get(potentiallyNewReport).addAll(wrappersToAdd.get(potentiallyNewReport));
         }
-
     }
 }

vcell-cli/src/main/java/org/vcell/cli/run/hdf5/Hdf5File.java

@@ -27,10 +27,11 @@
 public class Hdf5File {
     // NB: Hdf5 group management is ***important***.
     private final static Logger logger = LogManager.getLogger(Hdf5File.class);
+    private static boolean needToCreateFile = true;
 
     final private int H5F_ACC_TRUNC = HDF5Constants.H5F_ACC_TRUNC;
     final private int H5P_DEFAULT = HDF5Constants.H5P_DEFAULT;
-    final private int H5F_ACC_EXCL = HDF5Constants.H5F_ACC_EXCL;
+    final private int H5F_ACC_RDWR = HDF5Constants.H5F_ACC_RDWR;
     final private int H5T_C_S1 = HDF5Constants.H5T_C_S1;
     final private int H5I_INVALID_HID = HDF5Constants.H5I_INVALID_HID;
     final private int H5T_VARIABLE = HDF5Constants.H5T_VARIABLE;
@@ -42,8 +43,6 @@ public class Hdf5File {
     private int fileId;
     private boolean isOpen, allowExceptions;
 
-
-    // TODO: create actual java Bimap implementation(s) with accompaning java interface (good intro task for new appdevI hires?)
     private Map<Integer, String> idToPathMap;
     private Map<String, Integer> pathToIdMap;
     private Map<Integer, Integer> datasetToDataspaceMap;
@@ -56,12 +55,13 @@ private Hdf5File(){
         this.idToPathMap = new HashMap<Integer, String>();
         this.pathToIdMap = new HashMap<String, Integer>();
         this.datasetToDataspaceMap = new HashMap<>();
+        Hdf5File.needToCreateFile = false;
     }
 
     /**
      * Creates an Hdf5File named "reports.h5" in the provided directory, and will throw exceptions where c-style error codes would be returned.
      * 
-     * @param parentDir the directory to put the Hdf5 file inside of.
+     * @param parentDir the directory to put the Hdf5 file inside.
      */
     public Hdf5File(File parentDir) { //"/home/ldrescher/VCell/hdf5Rebuild/testingDir"
         this(parentDir, true);
@@ -110,7 +110,7 @@ public Hdf5File(File parentDir, String filename, boolean allowExceptions){
      * @throws IOException
      */
     public void open() throws HDF5Exception, IOException {
-        this.open(true);
+        this.open(Hdf5File.needToCreateFile);
     }
 
     /**
@@ -123,11 +123,12 @@ public void open() throws HDF5Exception, IOException {
      */
     public int open(boolean overwrite) throws HDF5Exception, IOException {
         String path = this.javaFileTarget.getCanonicalPath();
-        int accessMod = overwrite ? H5F_ACC_TRUNC: H5F_ACC_EXCL;
-        this.fileId = H5.H5Fcreate(path, accessMod, H5P_DEFAULT, H5P_DEFAULT);
+        if (overwrite) this.fileId = H5.H5Fopen(path, H5F_ACC_RDWR, H5P_DEFAULT);
+        else this.fileId = H5.H5Fcreate(path, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
         if (this.fileId < 0){
-            String message = "HDF5 File could not be created [H5Fcreate]; HDF5 files can not be generated.";
-            HDF5Exception e = new HDF5Exception(message); // nvestigate if Hdf5Exception would be more appropriate
+            String typeOfOperation = overwrite? "opened [H5Fopen]" : "created [H5Fopened]";
+            String message = "HDF5 File could not be " + typeOfOperation + "; Something is preventing this.";
+            HDF5Exception e = new HDF5Exception(message); // investigate if Hdf5Exception would be more appropriate
             logger.warn("Hdf5 error occured", e);
             if (this.allowExceptions) throw e;
         }