Add action to check conformance with GPAD ShEx on each change.

minerva-converter/src/main/java/org/geneontology/minerva/legacy/sparql/GPADSPARQLExport.java

@@ -12,8 +12,8 @@
 import org.apache.jena.rdf.model.Resource;
 import org.apache.jena.sparql.core.Var;
 import org.apache.jena.sparql.engine.binding.Binding;
+import org.apache.jena.sparql.engine.binding.BindingBuilder;
 import org.apache.jena.sparql.engine.binding.BindingFactory;
-import org.apache.jena.sparql.engine.binding.BindingMap;
 import org.apache.jena.vocabulary.RDF;
 import org.apache.log4j.Logger;
 import org.geneontology.minerva.curie.CurieHandler;
@@ -288,11 +288,11 @@ private Map<Triple, Set<GPADEvidence>> evidencesForFacts(Set<Triple> facts, Mode
 
     @SafeVarargs
     private final Binding createBinding(Pair<Var, Node>... bindings) {
-        BindingMap map = BindingFactory.create();
+        BindingBuilder builder = BindingFactory.builder();
         for (Pair<Var, Node> binding : bindings) {
-            map.add(binding.getLeft(), binding.getRight());
+            builder.add(binding.getLeft(), binding.getRight());
         }
-        return map;
+        return builder.build();
     }
 
     private Set<AnnotationExtension> possibleExtensions(Set<BasicGPADData> basicAnnotations, Model model) {

minerva-converter/src/test/java/org/geneontology/minerva/legacy/sparql/GPADSPARQLTest.java

@@ -5,7 +5,7 @@
 import org.apache.jena.rdf.model.Model;
 import org.apache.jena.rdf.model.ModelFactory;
 import org.apache.jena.reasoner.rulesys.Rule;
-import org.apache.jena.system.JenaSystem;
+import org.apache.jena.sys.JenaSystem;
 import org.geneontology.jena.OWLtoRules;
 import org.geneontology.minerva.curie.DefaultCurieHandler;
 import org.geneontology.rules.engine.RuleEngine;

minerva-core/pom.xml

@@ -54,6 +54,10 @@
             <artifactId>apache-jena-libs</artifactId>
             <type>pom</type>
         </dependency>
+        <dependency>
+            <groupId>org.apache.jena</groupId>
+            <artifactId>jena-commonsrdf</artifactId>
+        </dependency>
         <dependency>
             <groupId>net.sf.trove4j</groupId>
             <artifactId>trove4j</artifactId>
@@ -73,6 +77,10 @@
                     <groupId>org.apache.jena</groupId>
                     <artifactId>*</artifactId>
                 </exclusion>
+                <exclusion>  <!-- declare the exclusion here -->
+                    <groupId>org.apache.commons</groupId>
+                    <artifactId>commons-rdf-jena</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>
@@ -92,6 +100,11 @@
             <artifactId>commons-math3</artifactId>
             <version>3.6.1</version>
         </dependency>
+        <dependency>
+            <groupId>commons-codec</groupId>
+            <artifactId>commons-codec</artifactId>
+            <version>1.15</version>
+        </dependency>
         <dependency>
             <groupId>org.obolibrary.robot</groupId>
             <artifactId>robot-core</artifactId>

minerva-core/src/main/java/org/geneontology/minerva/BlazegraphMolecularModelManager.java

@@ -7,11 +7,18 @@
 import info.aduna.iteration.Iterations;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.jena.graph.Graph;
+import org.apache.jena.shex.ShapeMap;
+import org.apache.jena.shex.ShexReport;
+import org.apache.jena.shex.ShexSchema;
+import org.apache.jena.shex.ShexValidator;
+import org.apache.jena.sys.JenaSystem;
 import org.apache.log4j.Logger;
 import org.geneontology.minerva.MolecularModelManager.UnknownIdentifierException;
 import org.geneontology.minerva.curie.CurieHandler;
 import org.geneontology.minerva.util.AnnotationShorthand;
 import org.geneontology.minerva.util.BlazegraphMutationCounter;
+import org.geneontology.minerva.util.JenaOwlTool;
 import org.geneontology.minerva.util.ReverseChangeGenerator;
 import org.openrdf.model.*;
 import org.openrdf.model.impl.URIImpl;
@@ -22,6 +29,7 @@
 import org.openrdf.repository.RepositoryException;
 import org.openrdf.repository.RepositoryResult;
 import org.openrdf.rio.*;
+import org.openrdf.rio.helpers.BasicWriterSettings;
 import org.openrdf.rio.helpers.RDFHandlerBase;
 import org.openrdf.rio.helpers.StatementCollector;
 import org.semanticweb.owlapi.apibinding.OWLManager;
@@ -67,6 +75,7 @@ public class BlazegraphMolecularModelManager<METADATA> extends CoreMolecularMode
     public BlazegraphMolecularModelManager(OWLOntology tbox, CurieHandler curieHandler, String modelIdPrefix, @Nonnull String pathToJournal, String pathToExportFolder, String pathToOntologyJournal, boolean downloadOntologyJournal)
             throws OWLOntologyCreationException, IOException {
         super(tbox, pathToOntologyJournal, downloadOntologyJournal);
+        JenaSystem.init();
         if (curieHandler == null) {
             LOG.error("curie handler required for blazegraph model manager startup ");
             System.exit(-1);
@@ -195,23 +204,14 @@ public void saveModel(ModelContainer m)
         final OWLOntologyManager manager = ont.getOWLOntologyManager();
         List<OWLOntologyChange> changes = preSaveFileHandler(ont);
         synchronized (ont) {
-            try {
-                this.writeModelToDatabase(ont, modelId);
-                // reset modified flag for abox after successful save
-                m.setAboxModified(false);
-                // dump stored model to export file
-                if (this.pathToExportFolder != null) {
-                    File folder = new File(this.pathToExportFolder);
-                    dumpStoredModel(modelId, folder);
-                }
-            } finally {
-                if (changes != null) {
-                    List<OWLOntologyChange> invertedChanges = ReverseChangeGenerator
-                            .invertChanges(changes);
-                    if (invertedChanges != null && !invertedChanges.isEmpty()) {
-                        manager.applyChanges(invertedChanges);
-                    }
-                }
+            manager.applyChanges(changes);
+            this.writeModelToDatabase(ont, modelId);
+            // reset modified flag for abox after successful save
+            m.setAboxModified(false);
+            // dump stored model to export file
+            if (this.pathToExportFolder != null) {
+                File folder = new File(this.pathToExportFolder);
+                dumpStoredModel(modelId, folder);
             }
         }
     }
@@ -241,14 +241,10 @@ private void writeModelToDatabase(OWLOntology model, IRI modelId) throws Reposit
     }
 
     private List<OWLOntologyChange> preSaveFileHandler(OWLOntology model) throws UnknownIdentifierException {
-        List<OWLOntologyChange> allChanges = null;
+        List<OWLOntologyChange> allChanges = new ArrayList<>();
         for (PreFileSaveHandler handler : preFileSaveHandlers) {
             List<OWLOntologyChange> changes = handler.handle(model);
             if (changes != null && !changes.isEmpty()) {
-                if (allChanges == null) {
-                    allChanges = new ArrayList<OWLOntologyChange>(
-                            changes.size());
-                }
                 allChanges.addAll(changes);
             }
         }
@@ -994,4 +990,29 @@ public int addTaxonToDatabaseWithSparql(IRI model_iri, IRI taxon_iri) throws Rep
         return changes;
     }
 
+    public static class ShexValidatingPreSaveFileHandler implements PreFileSaveHandler {
+
+        private final ShexSchema schema;
+        private final ShapeMap shapeMap;
+        private final OWLAnnotationProperty conformanceProperty;
+        public ShexValidatingPreSaveFileHandler(ShexSchema schema, ShapeMap map, String propertyIRI) {
+            this.schema = schema;
+            this.shapeMap = map;
+            this.conformanceProperty = OWLManager.getOWLDataFactory().getOWLAnnotationProperty(IRI.create(propertyIRI));
+        }
+
+        @Override
+        public List<OWLOntologyChange> handle(OWLOntology modelOnt) throws UnknownIdentifierException {
+            OWLDataFactory factory = OWLManager.getOWLDataFactory();
+            Graph graph = JenaOwlTool.getJenaModel(modelOnt).getGraph();
+            ShexReport report = ShexValidator.get().validate(graph, schema, shapeMap);
+            boolean conforms = report.conforms();
+            System.out.println("CONFORMS? " + conforms);
+            System.out.println(factory.getOWLAnnotation(conformanceProperty, factory.getOWLLiteral(conforms)));
+            OWLOntologyChange remove = new RemoveOntologyAnnotation(modelOnt, factory.getOWLAnnotation(conformanceProperty, factory.getOWLLiteral(!conforms)));
+            OWLOntologyChange add = new AddOntologyAnnotation(modelOnt, factory.getOWLAnnotation(conformanceProperty, factory.getOWLLiteral(conforms)));
+            return Arrays.asList(remove, add);
+        }
+    }
+
 }

minerva-core/src/main/java/org/geneontology/minerva/util/AnnotationShorthand.java

@@ -30,7 +30,8 @@ public enum AnnotationShorthand {
     deprecated(OWLRDFVocabulary.OWL_DEPRECATED.getIRI()), // model annotation to indicate deprecated models
     templatestate(IRI.create("http://geneontology.org/lego/templatestate"), "template"), // designate a model as a template
     modelstate(IRI.create("http://geneontology.org/lego/modelstate"), "state"),
-    wasDerivedFrom(IRI.create("http://www.w3.org/ns/prov#wasDerivedFrom"), "wasDerivedFrom");
+    wasDerivedFrom(IRI.create("http://www.w3.org/ns/prov#wasDerivedFrom"), "wasDerivedFrom"),
+    conformsToGPAD(IRI.create("http://geneontology.org/lego/conforms_to_gpad"), "conforms-to-gpad");
 
 
     private final IRI annotationProperty;

minerva-core/src/main/java/org/geneontology/minerva/validation/ShexValidator.java

@@ -14,13 +14,15 @@
 import fr.inria.lille.shexjava.validation.Status;
 import fr.inria.lille.shexjava.validation.Typing;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.rdf.api.RDF;
-import org.apache.commons.rdf.api.RDFTerm;
-import org.apache.commons.rdf.jena.JenaGraph;
-import org.apache.commons.rdf.jena.JenaRDF;
+import org.apache.commons.rdf.api.*;
 import org.apache.commons.rdf.simple.SimpleRDF;
+import org.apache.jena.commonsrdf.JenaCommonsRDF;
+import org.apache.jena.commonsrdf.JenaRDF;
+import org.apache.jena.commonsrdf.impl.JenaGraph;
 import org.apache.jena.query.*;
 import org.apache.jena.rdf.model.*;
+import org.apache.jena.rdf.model.Literal;
+import org.apache.jena.sys.JenaSystem;
 import org.apache.log4j.Logger;
 import org.geneontology.minerva.BlazegraphOntologyManager;
 import org.geneontology.minerva.curie.CurieHandler;
@@ -34,7 +36,9 @@
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
+import java.util.UUID;
 import java.util.concurrent.*;
+import java.util.stream.Stream;
 
 /**
  * @author bgood
@@ -67,6 +71,7 @@ public ShexValidator(File shex_schema_file, File shex_map_file, BlazegraphOntolo
     }
 
     public void init(File shex_schema_file, File shex_map_file, BlazegraphOntologyManager go_lego, CurieHandler curieHandler_) throws Exception {
+        JenaSystem.init();
         schema = GenParser.parseSchema(shex_schema_file.toPath());
         GoQueryMap = makeGoQueryMap(shex_map_file.getAbsolutePath());
         //tbox_reasoner = tbox_reasoner_;
@@ -115,9 +120,8 @@ public static Map<String, String> makeGoQueryMap(String shapemap_file) throws IO
     public ShexValidationReport runShapeMapValidation(Model test_model) {
         boolean explain = true;
         ShexValidationReport r = new ShexValidationReport();
-        JenaRDF jr = new JenaRDF();
         //this shex implementation likes to use the commons JenaRDF interface, nothing exciting here
-        JenaGraph shexy_graph = jr.asGraph(test_model);
+        Graph shexy_graph = JenaCommonsRDF.fromJena(test_model.getGraph());
         boolean all_good = true;
         try {
             Typing all_typed = runRefineWithTimeout(shexy_graph);
@@ -251,7 +255,7 @@ public ShexValidationReport runShapeMapValidationWithRecursiveSingleNodeValidati
         ShexValidationReport r = new ShexValidationReport();
         JenaRDF jr = new JenaRDF();
         //this shex implementation likes to use the commons JenaRDF interface, nothing exciting here
-        JenaGraph shexy_graph = jr.asGraph(test_model);
+        Graph shexy_graph = JenaCommonsRDF.fromJena(test_model.getGraph());
         //recursive only checks the focus node against the chosen shape.
         RecursiveValidationWithMemorization shex_model_validator = new RecursiveValidationWithMemorization(schema, shexy_graph);
         //for each shape in the query map (e.g. MF, BP, CC, etc.)
@@ -391,7 +395,7 @@ private Map<Resource, Set<String>> getShapesToTestForEachResource(Model test_mod
         return node_s_shapes;
     }
 
-    private Typing runRefineWithTimeout(JenaGraph shexy_graph) {
+    private Typing runRefineWithTimeout(Graph shexy_graph) {
         final ExecutorService service = Executors.newSingleThreadExecutor();
         try {
             final Future<Typing> f = service.submit(() -> {

minerva-server/src/main/java/org/geneontology/minerva/server/StartUpTool.java

@@ -1,12 +1,16 @@
 package org.geneontology.minerva.server;
 
+import org.apache.jena.shex.ShapeMap;
+import org.apache.jena.shex.ShexSchema;
+import org.apache.jena.shex.parser.ShExC;
 import org.apache.log4j.Logger;
 import org.eclipse.jetty.server.HttpConfiguration;
 import org.eclipse.jetty.server.HttpConnectionFactory;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.ServerConnector;
 import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.servlet.ServletHolder;
+import org.geneontology.minerva.BlazegraphMolecularModelManager;
 import org.geneontology.minerva.MinervaOWLGraphWrapper;
 import org.geneontology.minerva.ModelReaderHelper;
 import org.geneontology.minerva.UndoAwareMolecularModelManager;
@@ -93,6 +97,10 @@ public static class MinervaStartUpConfig {
         public MinervaShexValidator shex;
         public String pathToOntologyJournal;
 
+        public String pathToGPADSheX;
+
+        public String pathToGPADShapeMap;
+
     }
 
     public static void main(String[] args) throws Exception {
@@ -172,6 +180,10 @@ else if (opts.nextEq("--golr-labels")) {
                 conf.sparqlEndpointTimeout = Integer.parseInt(opts.nextOpt());
             } else if (opts.nextEq("--ontojournal")) {
                 conf.pathToOntologyJournal = opts.nextOpt();
+            } else if (opts.nextEq("--gpad-shex")) {
+                conf.pathToGPADSheX = opts.nextOpt();
+            } else if (opts.nextEq("--gpad-shape-map")) {
+                conf.pathToGPADShapeMap = opts.nextOpt();
             } else {
                 break;
             }
@@ -340,6 +352,11 @@ public static Server startUp(final MinervaStartUpConfig conf)
                 conf.curieHandler, conf.modelIdPrefix, conf.journalFile, conf.exportFolder, conf.pathToOntologyJournal, true);
         // set pre and post file handlers
         models.addPostLoadOntologyFilter(ModelReaderHelper.INSTANCE);
+        if (conf.pathToGPADSheX != null && conf.pathToGPADShapeMap != null) {
+            ShexSchema schema = ShExC.parse(conf.pathToGPADSheX);
+            ShapeMap shapeMap = ShExC.parseShapeMap(conf.pathToGPADShapeMap);
+            models.addPreFileSaveHandler(new BlazegraphMolecularModelManager.ShexValidatingPreSaveFileHandler(schema, shapeMap, "http://geneontology.org/lego/conforms_to_gpad"));
+        }
         //	conf.shex.tbox_reasoner = models.getTbox_reasoner();
         conf.shex.setGo_lego_repo(models.getGolego_repo());
         conf.shex.curieHandler = conf.curieHandler;

minerva-server/src/main/java/org/geneontology/minerva/server/handler/ModelSearchHandler.java

@@ -10,6 +10,7 @@
 import org.openrdf.repository.RepositoryException;
 import org.semanticweb.owlapi.model.IRI;
 
+import javax.annotation.Nullable;
 import javax.ws.rs.*;
 import javax.ws.rs.core.MediaType;
 import java.io.IOException;
@@ -89,18 +90,21 @@ static public class ModelMeta {
         private String date;
         private String title;
         private String state;
+        @SerializedName("conforms-to-gpad")
+        private Boolean conformsToGPAD;
         private Set<String> contributors;
         private Set<String> groups;
         private HashMap<String, Set<String>> query_match;
 
         @SerializedName("modified-p")
         private boolean modified;
 
-        public ModelMeta(String id, String date, String title, String state, Set<String> contributors, Set<String> groups, boolean modified) {
+        public ModelMeta(String id, String date, String title, String state, @Nullable Boolean conformsToGPAD, Set<String> contributors, Set<String> groups, boolean modified) {
             this.id = id;
             this.date = date;
             this.title = title;
             this.state = state;
+            this.conformsToGPAD = conformsToGPAD;
             this.contributors = contributors;
             this.groups = groups;
             this.modified = modified;
@@ -150,6 +154,15 @@ public void setState(String state) {
             this.state = state;
         }
 
+        @Nullable
+        public Boolean getConformsToGPAD() {
+            return this.conformsToGPAD;
+        }
+
+        public void setConformsToGPAD(Boolean conforms) {
+            this.conformsToGPAD = conforms;
+        }
+
         public Set<String> getContributors() {
             return contributors;
         }
@@ -454,7 +467,7 @@ public ModelSearchResult search(Set<String> taxa,
         String group_by_constraint = "GROUP BY ?id";
         //default return block
         //TODO investigate need to add DISTINCT to GROUP_CONCAT here
-        String return_block = "?id (MIN(?date) AS ?mindate) (MIN(?title) AS ?mintitle) (MIN(?state) AS ?minstate) <ind_return_list> (GROUP_CONCAT(DISTINCT ?contributor;separator=\";\") AS ?contributors) (GROUP_CONCAT(DISTINCT ?group;separator=\";\") AS ?groups)";
+        String return_block = "?id (MIN(?date) AS ?mindate) (MIN(?title) AS ?mintitle) (MIN(?state) AS ?minstate) (MIN(?conformsToGPAD) AS ?minConformsToGPAD) <ind_return_list> (GROUP_CONCAT(DISTINCT ?contributor;separator=\";\") AS ?contributors) (GROUP_CONCAT(DISTINCT ?group;separator=\";\") AS ?groups)";
         if (count != null) {
             return_block = "(count(distinct ?id) as ?count)";
             limit_constraint = "";
@@ -525,6 +538,12 @@ public ModelSearchResult search(Set<String> taxa,
                     if (state_binding != null) {
                         state = state_binding.getValue().stringValue();
                     }
+                    Binding conformanceBinding = bs.getBinding("minConformsToGPAD");
+                    Boolean conformance = null;
+                    if (conformanceBinding != null) {
+                        System.out.println("CONFORMANCE: " + conformanceBinding.getValue().stringValue());
+                        conformance = Boolean.valueOf(conformanceBinding.getValue().stringValue());
+                    }
                     Binding group_binding = bs.getBinding("groups");
                     String groups_ = "";
                     if (group_binding != null) {
@@ -539,7 +558,7 @@ public ModelSearchResult search(Set<String> taxa,
                     if (mm == null) {
                         //look up model in in-memory cache to check edit state
                         boolean is_modified = m3.isModelModified(model_iri);
-                        mm = new ModelMeta(model_curie, date, title, state, contributors, groups, is_modified);
+                        mm = new ModelMeta(model_curie, date, title, state, conformance, contributors, groups, is_modified);
                     }
                     //matching
                     for (String ind : ind_return.keySet()) {

minerva-server/src/main/resources/ModelSearchQueryTemplate.rq

@@ -21,6 +21,7 @@ GRAPH ?id {
 <http://purl.org/dc/elements/1.1/contributor> ?contributor ;
 optional{?id <http://purl.org/pav/providedBy> ?group } .
 optional{?id lego:modelstate ?state } .
+optional{?id lego:conforms_to_gpad ?conformsToGPAD}
 <types>
 <state_constraint>
 <contributor_constraint>