Improve provenance generation of execution

README.md

@@ -6,7 +6,7 @@
 
 ### Docker
 
-You must install [Docker](https://www.docker.com/) and [docker-compose](https://docs.docker.com/compose/install/).
+You must install [Docker](https://www.docker.com/) and [docker-compose](https://docs.docker.com/compose/install/). 
 
 Clone the repository
 

docs/configuration.md

@@ -82,3 +82,46 @@ execution =
         }
     }
 ```
+
+## Publisher
+
+The publisher section of the configuration file describes how the Wings system shares data, execution, and provenance. The following properties are available:
+
+```
+    publisher =
+    {
+        file-store=
+        {
+            url = "https://publisher.mint.isi.edu";
+            type = "FILE_SYSTEM";
+        }
+        triple-store = {
+            export-name = "exportTest";
+            export-url = "https://opmw.org/";
+            publish = https://endpoint.mint.isi.edu/provenance/data;
+            query = https://endpoint.mint.isi.edu/provenance/query;
+            domains-directory = /opt/wings/storage/default;
+        }
+    }
+```
+
+### File store
+
+The file store section of the configuration file describes how the Wings system shares data. The following properties are available:
+
+| Name                      | Description                       | Default Value |
+| ------------------------- | --------------------------------- | ------------- |
+| publisher.file-store.url  | URI where the files are published |               |
+| publisher.file-store.type | Type of file store (FILE_SYSTEM)  | FILE_SYSTEM   |
+
+### Triple store
+
+The triple store section of the configuration file describes how the Wings system shares execution and provenance. The following properties are available:
+
+| Name                                     | Description                            | Default Value                                  |
+| ---------------------------------------- | -------------------------------------- | ---------------------------------------------- |
+| publisher.triple-store.export-name       | Name of the export (exportTest)        | exportTest                                     |
+| publisher.triple-store.export-url        | URI where the files are published      | https://opmw.org/                              |
+| publisher.triple-store.publish           | URI where the provenance is published  | https://endpoint.mint.isi.edu/provenance/data  |
+| publisher.triple-store.query             | URI where the provenance is queried    | https://endpoint.mint.isi.edu/provenance/query |
+| publisher.triple-store.domains-directory | Directory where the domains are stored | /opt/wings/storage/default                     |

engines/oodt/adapter/src/main/java/edu/isi/wings/catalog/component/api/impl/oodt/ComponentCreationFM.java

@@ -68,10 +68,14 @@ public class ComponentCreationFM implements ComponentCreationAPI {
 
   OntFactory ontologyFactory;
 
-  private static HashMap<String, String> compHolders =
-    new HashMap<String, String>();
-  private static HashMap<String, Metadata> compMeta =
-    new HashMap<String, Metadata>();
+  private static HashMap<String, String> compHolders = new HashMap<
+    String,
+    String
+  >();
+  private static HashMap<String, Metadata> compMeta = new HashMap<
+    String,
+    Metadata
+  >();
 
   public ComponentCreationFM(Properties props) {
     this.fmurl = props.getProperty("oodt.fmurl");
@@ -102,8 +106,10 @@ public ComponentTree getComponentHierarchy(boolean details) {
     String rootid = this.absurl + "#Component";
     ComponentHolder rootholder = new ComponentHolder(rootid);
     ComponentTreeNode rootnode = new ComponentTreeNode(rootholder);
-    HashMap<String, ComponentTreeNode> tnmap =
-      new HashMap<String, ComponentTreeNode>();
+    HashMap<String, ComponentTreeNode> tnmap = new HashMap<
+      String,
+      ComponentTreeNode
+    >();
     HashMap<String, String> pmap = this.curatorApi.getParentTypeMap();
     try {
       for (ProductType ptype : this.fmclient.getProductTypes()) {

engines/oodt/adapter/src/main/java/edu/isi/wings/catalog/data/api/impl/oodt/DataCreationFM.java

@@ -59,8 +59,10 @@ public class DataCreationFM implements DataCreationAPI {
   String liburl;
   String onturl;
 
-  private static HashMap<String, String> prodTypes =
-    new HashMap<String, String>();
+  private static HashMap<String, String> prodTypes = new HashMap<
+    String,
+    String
+  >();
 
   public DataCreationFM(Properties props) {
     this.fmurl = props.getProperty("oodt.fmurl");

engines/oodt/adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/oodt/OODTExecutionEngine.java

@@ -242,8 +242,10 @@ public ExecutionMonitoringThread(
     public void run() {
       planexe.onStart(this.logger);
       try {
-        HashMap<String, RuntimeInfo.Status> jobstatus =
-          new HashMap<String, RuntimeInfo.Status>();
+        HashMap<String, RuntimeInfo.Status> jobstatus = new HashMap<
+          String,
+          RuntimeInfo.Status
+        >();
 
         int osleeptime = 1000;
         int maxsleeptime = 4 * osleeptime;

engines/oodt/adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/oodt/OODTWorkflowAdapter.java

@@ -80,8 +80,10 @@ public OODTWorkflowAdapter(
   public Workflow runWorkflow(RuntimePlan planexe, Metadata meta)
     throws Exception {
     // Get List of all Jobs
-    HashMap<String, WorkflowTask> tasksById =
-      new HashMap<String, WorkflowTask>();
+    HashMap<String, WorkflowTask> tasksById = new HashMap<
+      String,
+      WorkflowTask
+    >();
     HashMap<String, String> opProducers = new HashMap<String, String>();
     HashMap<String, String> taskJobs = new HashMap<String, String>();
     for (RuntimeStep stepexe : planexe.getQueue().getAllSteps()) {
@@ -117,8 +119,10 @@ public Workflow runWorkflow(RuntimePlan planexe, Metadata meta)
     }
 
     // Get Parent Child Relationship between jobs
-    HashMap<String, ArrayList<String>> parents =
-      new HashMap<String, ArrayList<String>>();
+    HashMap<String, ArrayList<String>> parents = new HashMap<
+      String,
+      ArrayList<String>
+    >();
     for (RuntimeStep stepexe : planexe.getQueue().getAllSteps()) {
       ExecutionStep step = stepexe.getStep();
       ArrayList<String> chparents = parents.get(stepexe.getName());
@@ -132,8 +136,10 @@ public Workflow runWorkflow(RuntimePlan planexe, Metadata meta)
     }
 
     // Arrange Jobs into Job Levels
-    TreeMap<Integer, ArrayList<String>> jobLevels =
-      new TreeMap<Integer, ArrayList<String>>();
+    TreeMap<Integer, ArrayList<String>> jobLevels = new TreeMap<
+      Integer,
+      ArrayList<String>
+    >();
     for (String jobid : tasksById.keySet()) {
       Integer lvl = new Integer(getJobLevel(jobid, 0, parents));
       ArrayList<String> lvljobs;

engines/oodt/adapter/src/main/java/edu/isi/wings/util/oodt/CurationServiceAPI.java

@@ -49,10 +49,10 @@ private String query(String method, String op, Object... args) {
       String params = "policy=" + URLEncoder.encode(this.policy, "UTF-8");
       for (int i = 0; i < args.length; i += 2) {
         params +=
-          "&" +
-          args[i] +
-          "=" +
-          URLEncoder.encode(args[i + 1].toString(), "UTF-8");
+        "&" +
+        args[i] +
+        "=" +
+        URLEncoder.encode(args[i + 1].toString(), "UTF-8");
       }
 
       URL urlobj = new URL(url);
@@ -117,7 +117,7 @@ public boolean addElementsForProductType(
   ) {
     String elementIds = "";
     for (Element element : elementList) elementIds +=
-      (elementIds != "" ? "," : "") + element.getElementId();
+    (elementIds != "" ? "," : "") + element.getElementId();
     String result =
       this.query(
           "POST",
@@ -169,7 +169,7 @@ public boolean removeElementsForProductType(
   ) {
     String elementIds = "";
     for (Element element : elementList) elementIds +=
-      (elementIds != "" ? "," : "") + element.getElementId();
+    (elementIds != "" ? "," : "") + element.getElementId();
     String result =
       this.query(
           "DELETE",

engines/oodt/wmservices/src/main/java/org/apache/oodt/cas/wmservices/client/WmServicesClient.java

@@ -116,7 +116,7 @@ private String query(String method, String op, Object... args) {
       for (int i = 0; i < args.length; i += 2) {
         if (i > 0) params += "&";
         params +=
-          args[i] + "=" + URLEncoder.encode(args[i + 1].toString(), "UTF-8");
+        args[i] + "=" + URLEncoder.encode(args[i + 1].toString(), "UTF-8");
       }
       if ("GET".equals(method)) {
         URL urlobj = new URL(url + "?" + params);

planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/ComponentCreationKB.java

@@ -306,9 +306,9 @@ protected boolean incrementComponentVersion(String cid, KBAPI writerkb) {
       KBObject compobj = kb.getIndividual(cid);
       KBObject versionProp = kb.getProperty(this.pcns + "hasVersion");
       KBObject versionVal = kb.getPropertyValue(compobj, versionProp);
-      int currentVersion = (Integer) (
-        versionVal.getValue() != null ? versionVal.getValue() : 0
-      );
+      int currentVersion = (Integer) (versionVal.getValue() != null
+          ? versionVal.getValue()
+          : 0);
 
       int newVersion = currentVersion + 1;
       KBObject cobj = writerkb.getResource(cid);

planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/ComponentKB.java

@@ -820,11 +820,9 @@ public Component getComponent(String cid, boolean details) {
   protected int getComponentVersion(KBObject compobj) {
     KBObject versionProp = kb.getProperty(this.pcns + "hasVersion");
     KBObject versionVal = kb.getPropertyValue(compobj, versionProp);
-    return (Integer) (
-      (versionVal != null && versionVal.getValue() != null)
+    return (Integer) ((versionVal != null && versionVal.getValue() != null)
         ? versionVal.getValue()
-        : 0
-    );
+        : 0);
   }
 
   protected ArrayList<KBObject> getComponentInputs(KBObject compobj) {

planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/ComponentReasoningKB.java

@@ -53,14 +53,22 @@ public class ComponentReasoningKB
   private boolean useRules = true;
 
   private HashMap<String, Component> ccache = new HashMap<String, Component>();
-  private HashMap<String, KBRuleList> rulescache =
-    new HashMap<String, KBRuleList>();
-  private HashMap<String, ArrayList<String>> abscache =
-    new HashMap<String, ArrayList<String>>();
-  private HashMap<String, ArrayList<KBObject>> classcache =
-    new HashMap<String, ArrayList<KBObject>>();
-  private HashMap<String, ArrayList<KBTriple>> kbcache =
-    new HashMap<String, ArrayList<KBTriple>>();
+  private HashMap<String, KBRuleList> rulescache = new HashMap<
+    String,
+    KBRuleList
+  >();
+  private HashMap<String, ArrayList<String>> abscache = new HashMap<
+    String,
+    ArrayList<String>
+  >();
+  private HashMap<String, ArrayList<KBObject>> classcache = new HashMap<
+    String,
+    ArrayList<KBObject>
+  >();
+  private HashMap<String, ArrayList<KBTriple>> kbcache = new HashMap<
+    String,
+    ArrayList<KBTriple>
+  >();
 
   private ArrayList<KBTriple> metricTriples;
   private ArrayList<KBObject> metricProps;
@@ -897,8 +905,10 @@ public ArrayList<ComponentPacket> findOutputDataPredictedDescriptions(
           );
 
       // Keep a map of variable object to variable name
-      HashMap<Variable, String> variableNameMap =
-        new HashMap<Variable, String>();
+      HashMap<Variable, String> variableNameMap = new HashMap<
+        Variable,
+        String
+      >();
 
       for (String rolestr : sRoleMap.keySet()) {
         Variable var = sRoleMap.get(rolestr);
@@ -926,8 +936,10 @@ public ArrayList<ComponentPacket> findOutputDataPredictedDescriptions(
 
       // Get a mapping of ArgID's to arg for the Component
       // Also note which roles are inputs
-      HashMap<String, ComponentRole> argMaps =
-        new HashMap<String, ComponentRole>();
+      HashMap<String, ComponentRole> argMaps = new HashMap<
+        String,
+        ComponentRole
+      >();
       HashMap<String, Boolean> sInputRoles = new HashMap<String, Boolean>();
       for (ComponentRole role : comp.getInputs()) {
         argMaps.put(role.getRoleName(), role);
@@ -1199,8 +1211,9 @@ public ArrayList<ComponentPacket> findOutputDataPredictedDescriptions(
 
           // Marking all compatible inputs to be passed through to the output.
           // Note: If not all outputs have a matching input, then component is still run and not skipped
-          ArrayList<ComponentRole> matchingInputs =
-            new ArrayList<ComponentRole>();
+          ArrayList<ComponentRole> matchingInputs = new ArrayList<
+            ComponentRole
+          >();
           for (ComponentRole inrole : comp.getInputs()) {
             if (!inrole.isParam()) {
               if (checkTypeCompatibility(varclasses, inrole.getID())) {
@@ -1261,11 +1274,9 @@ public ArrayList<ComponentPacket> findOutputDataPredictedDescriptions(
       for (Variable var : sRoleMap.values()) {
         if (
           var.isParameterVariable() &&
-          (
-            noParamBindings.containsKey(var.getID()) ||
+          (noParamBindings.containsKey(var.getID()) ||
             var.getBinding() == null ||
-            var.getBinding().getValue() == null
-          )
+            var.getBinding().getValue() == null)
         ) {
           KBObject varobj = tkb.getResource(variableNameMap.get(var));
           KBObject origvarobj = tkb.getResource(var.getID());

planner/src/main/java/edu/isi/wings/catalog/component/classes/requirements/Dependency.java

@@ -70,12 +70,12 @@ public boolean checkDependency(KBAPI api, KBObject obj) {
       if (
         this.operator == Operator.GREATER_THAN &&
         Double.parseDouble(obj.getValueAsString()) >
-        Double.parseDouble(this.desiredValue.toString())
+          Double.parseDouble(this.desiredValue.toString())
       ) return true;
       if (
         this.operator == Operator.LESS_THAN &&
         Double.parseDouble(obj.getValueAsString()) <
-        Double.parseDouble(this.desiredValue.toString())
+          Double.parseDouble(this.desiredValue.toString())
       ) return true;
     }
     return false;

planner/src/main/java/edu/isi/wings/catalog/data/api/impl/kb/DataCreationKB.java

@@ -137,8 +137,10 @@ public ArrayList<String> getAllDatatypeIds() {
 
   @Override
   public HashMap<String, ArrayList<String>> getAllDatatypeDatasets() {
-    HashMap<String, ArrayList<String>> typedata =
-      new HashMap<String, ArrayList<String>>();
+    HashMap<String, ArrayList<String>> typedata = new HashMap<
+      String,
+      ArrayList<String>
+    >();
     String query =
       "SELECT ?s ?type\n" +
       "WHERE {\n" +

planner/src/main/java/edu/isi/wings/catalog/data/classes/MetadataProperty.java

@@ -74,15 +74,15 @@ public void setRange(String id) {
   public String toString() {
     String str = "";
     str +=
-      "\n" +
-      getName() +
-      "(" +
-      type +
-      ")\nDomains:" +
-      domains +
-      "\nRange:" +
-      range +
-      "\n";
+    "\n" +
+    getName() +
+    "(" +
+    type +
+    ")\nDomains:" +
+    domains +
+    "\nRange:" +
+    range +
+    "\n";
     return str;
   }
 }

planner/src/main/java/edu/isi/wings/catalog/resource/api/impl/kb/ResourceKB.java

@@ -45,8 +45,10 @@ public class ResourceKB extends TransactionsJena implements ResourceAPI {
   public String onturl, liburl;
   private String tdbRepository;
 
-  private static HashMap<String, Boolean> initializedLibrary =
-    new HashMap<String, Boolean>();
+  private static HashMap<String, Boolean> initializedLibrary = new HashMap<
+    String,
+    Boolean
+  >();
 
   private KBAPI ontkb, libkb;
 
@@ -398,8 +400,9 @@ public ArrayList<SoftwareEnvironment> getAllSoftwareEnvironment() {
     this.start_read();
     boolean batched = this.start_batch_operation();
 
-    ArrayList<SoftwareEnvironment> environment =
-      new ArrayList<SoftwareEnvironment>();
+    ArrayList<SoftwareEnvironment> environment = new ArrayList<
+      SoftwareEnvironment
+    >();
     for (KBObject swobj : this.getInstancesOfClass(
         this.cmap.get("SoftwareGroup"),
         this.libkb
@@ -703,8 +706,10 @@ public void setMachineWhitelist(ArrayList<String> whitelist) {
   }
 
   public void copyFrom(ResourceAPI rc, ComponentCreationAPI cc) {
-    HashMap<String, SoftwareVersion> sws =
-      new HashMap<String, SoftwareVersion>();
+    HashMap<String, SoftwareVersion> sws = new HashMap<
+      String,
+      SoftwareVersion
+    >();
 
     this.start_write();
     boolean batched = this.start_batch_operation();