diff --git a/ontospy/VERSION.py b/ontospy/VERSION.py index 2eebbab4..705e23fc 100755 --- a/ontospy/VERSION.py +++ b/ontospy/VERSION.py @@ -8,9 +8,9 @@ # note: 1.8 major update with ontodocs split ################## -__version__ = "1.9.8.3" # Pypi latest: https://pypi.org/project/ontospy/ +__version__ = "1.9.9" # Pypi latest: https://pypi.org/project/ontospy/ -__copyright__ = "CopyRight (C) 2015-2018 by Michele Pasin" +__copyright__ = "CopyRight (C) 2015-2021 by Michele Pasin" __license__ = "GNU" __author__ = "Michele Pasin" __author_email__ = "michele dot pasin at gmail dot com" diff --git a/ontospy/__init__.py b/ontospy/__init__.py index 82d7bad0..bda5ce72 100755 --- a/ontospy/__init__.py +++ b/ontospy/__init__.py @@ -5,4 +5,4 @@ from .VERSION import __version__, VERSION from .core.ontospy import Ontospy -from .core.entities import RDF_Entity, Ontology, OntoClass, OntoProperty, OntoSKOSConcept, OntoShape +from .core.entities import RdfEntity, Ontology, OntoClass, OntoProperty, OntoSKOSConcept, OntoShape diff --git a/ontospy/cli.py b/ontospy/cli.py index b7758f3f..4e581cd6 100755 --- a/ontospy/cli.py +++ b/ontospy/cli.py @@ -197,6 +197,14 @@ def scan(ctx, sources=None, endpoint=False, raw=False, extra=False): help= 'THEME: select bootstrap style (only for the html-multi-page visualization). Default: simplex (random=use a random theme).' ) +@click.option( + '--preflabel', + help='PREF-LABEL: default value to use for entity titles (qname|label - default=qname).') +@click.option( + '--preflang', + help= + 'PREF-LANGUAGE: default language for multilingual strings (default=en).' +) @click.option( '--nobrowser', is_flag=True, @@ -218,6 +226,8 @@ def gendocs(ctx, type="", title="", theme="", + preflabel="", + preflang="", nobrowser=False, showthemes=False, showtypes=False): @@ -256,6 +266,12 @@ def gendocs(ctx, if theme and theme == "random": theme = random_theme() + if preflabel and preflabel not in ["label", "qname"]: + click.secho( + "WARNING: the valid preflabel options are either 'qname' or 'label' (= rdfs:label) only. Using defaults.", + fg="red") + preflabel = "qname" + if outputpath: if not (os.path.exists(outputpath)) or not (os.path.isdir(outputpath)): click.secho( @@ -283,6 +299,8 @@ def gendocs(ctx, title=title, viztype=type, theme=theme, + preflabel=preflabel, + preflang=preflang, verbose=verbose) if url and (not nobrowser): # open browser diff --git a/ontospy/core/actions.py b/ontospy/core/actions.py index 682f6787..81aa8ee3 100755 --- a/ontospy/core/actions.py +++ b/ontospy/core/actions.py @@ -636,6 +636,8 @@ def action_visualize(args, title="", viztype="", theme="", + preflabel="", + preflang="", verbose=False): """ export model into another format eg html, d3 etc... @@ -663,9 +665,9 @@ def action_visualize(args, # 2017-01-23: bypass pickled stuff as it has wrong counts etc.. # get ontospy graph printDebug("Loading graph...", dim=True) - g = Ontospy(ontouri, verbose=verbose) + g = Ontospy(ontouri, verbose=verbose, pref_title=preflabel, pref_lang=preflang) - # put in home folder by default: //files.. + # put viz in home folder by default: //files.. if not path: from os.path import expanduser home = expanduser("~") diff --git a/ontospy/core/entities.py b/ontospy/core/entities.py index 23d97342..e50555ed 100755 --- a/ontospy/core/entities.py +++ b/ontospy/core/entities.py @@ -13,7 +13,7 @@ from .utils import * -class RDF_Entity(object): +class RdfEntity(object): """ Pythonic representation of an RDF resource - normally not instantiated but used for inheritance purposes @@ -30,14 +30,17 @@ class RDF_Entity(object): _ids = count(0) def __repr__(self): - return "" % (self.uri) + return "" % (self.uri) def __init__(self, uri, rdftype=None, namespaces=None, ext_model=False, - is_Bnode=False): + is_Bnode=False, + pref_title="qname", + pref_lang="en", + ): """ Init ontology object. Load the graph in memory, then setup all necessary attributes. @@ -53,6 +56,9 @@ def __init__(self, self.locale = inferURILocalSymbol(self.uri)[0] self.ext_model = ext_model self.is_Bnode = is_Bnode + self._pref_title = pref_title + self._pref_lang = pref_lang + self.slug = None self.rdftype = rdftype self.triples = None @@ -109,7 +115,7 @@ def _buildGraph(self): for terzetto in self.triples: self.rdflib_graph.add(terzetto) - # methods added to RDF_Entity even though they apply only to some subs + # methods added to RdfEntity even though they apply only to some subs def ancestors(self, cl=None, noduplicates=True): """ returns all ancestors in the taxonomy """ @@ -169,7 +175,7 @@ def getValuesForProperty(self, aPropURIRef): aPropURIRef = rdflib.URIRef(aPropURIRef) return list(self.rdflib_graph.objects(None, aPropURIRef)) - def bestLabel(self, prefLanguage="en", qname_allowed=True, quotes=False): + def bestLabel(self, prefLanguage="", qname_allowed=True, quotes=False): """ facility for extrating the best available label for an entity @@ -179,6 +185,9 @@ def bestLabel(self, prefLanguage="en", qname_allowed=True, quotes=False): test = self.getValuesForProperty(rdflib.RDFS.label) out = "" + if not prefLanguage: + prefLanguage = self._pref_lang + if test: out = firstStringInList(test, prefLanguage) else: @@ -194,7 +203,7 @@ def bestLabel(self, prefLanguage="en", qname_allowed=True, quotes=False): else: return out - def bestDescription(self, prefLanguage="en", quotes=False): + def bestDescription(self, prefLanguage="", quotes=False): """ facility for extracting a human readable description for an entity """ @@ -204,6 +213,9 @@ def bestDescription(self, prefLanguage="en", quotes=False): rdflib.namespace.DC.description, rdflib.namespace.SKOS.definition ] + if not prefLanguage: + prefLanguage = self._pref_lang + for pred in test_preds: test = self.getValuesForProperty(pred) # printDebug(str(test), "red") @@ -214,8 +226,27 @@ def bestDescription(self, prefLanguage="en", quotes=False): return joinStringsInList(test, prefLanguage) return "" + @property + def title(self): + """Entity title - used for display purposes only. + Can be set by user once ontospy is created. + Values allowed: 'qname' or 'label' + + Defaults to 'qname'. + """ + + if self._pref_title == "qname": + out = self.qname + elif self._pref_title == "label": + out = self.bestLabel() + else: + return self.qname -class Ontology(RDF_Entity): + return out + + + +class Ontology(RdfEntity): """ Pythonic representation of an OWL ontology """ @@ -227,14 +258,17 @@ def __init__(self, uri, rdftype=None, namespaces=None, - prefPrefix="", - ext_model=False): + pref_prefix="", + ext_model=False, + pref_title="qname", + pref_lang="en", + ): """ Init ontology object. Load the graph in memory, then setup all necessary attributes. """ - super(Ontology, self).__init__(uri, rdftype, namespaces, ext_model) + super().__init__(uri, rdftype, namespaces, ext_model, pref_title=pref_title, pref_lang=pref_lang) # self.uri = uri # rdflib.Uriref - self.prefix = prefPrefix + self.prefix = pref_prefix self.slug = "ontology-" + slugify(self.qname) self.all_classes = [] self.all_properties = [] @@ -265,7 +299,7 @@ def stats(self): printDebug("Properties..: %d" % len(self.all_properties)) -class OntoClass(RDF_Entity): +class OntoClass(RdfEntity): """ Python representation of a generic class within an ontology. Includes methods for representing and querying RDFS/OWL classes @@ -280,13 +314,14 @@ class OntoClass(RDF_Entity): ] """ - def __init__(self, uri, rdftype=None, namespaces=None, ext_model=False): + def __init__(self, uri, rdftype=None, namespaces=None, + ext_model=False, pref_title="qname", pref_lang="en"): """ ... """ - super(OntoClass, self).__init__(uri, rdftype, namespaces, ext_model) + super().__init__(uri, rdftype, namespaces, ext_model, + pref_title=pref_title, pref_lang=pref_lang) self.slug = "class-" + slugify(self.qname) - self.domain_of = [] self.range_of = [] self.domain_of_inferred = [] @@ -308,7 +343,7 @@ def instances(self): # = all instances if self.sparqlHelper: qres = self.sparqlHelper.getClassInstances(self.uri) for uri in [x[0] for x in qres]: - instance = RDF_Entity(uri, self.uri, self.namespaces) + instance = RdfEntity(uri, self.uri, self.namespaces) instance.triples = self.sparqlHelper.entityTriples( instance.uri) instance._buildGraph() # force construction of mini graph @@ -345,7 +380,7 @@ def describe(self): # self.printGenericTree() -class OntoProperty(RDF_Entity): +class OntoProperty(RdfEntity): """ Python representation of a generic RDF/OWL property. @@ -357,11 +392,12 @@ class OntoProperty(RDF_Entity): """ - def __init__(self, uri, rdftype=None, namespaces=None, ext_model=False): + def __init__(self, uri, rdftype=None, namespaces=None, ext_model=False, + pref_title="qname", pref_lang="en"): """ ... """ - super(OntoProperty, self).__init__(uri, rdftype, namespaces, ext_model) + super().__init__(uri, rdftype, namespaces, ext_model, pref_title=pref_title, pref_lang=pref_lang) self.slug = "prop-" + slugify(self.qname) self.rdftype = inferMainPropertyType(rdftype) @@ -395,19 +431,19 @@ def describe(self): # self.printGenericTree() -class OntoSKOSConcept(RDF_Entity): +class OntoSKOSConcept(RdfEntity): """ Python representation of a generic SKOS concept within an ontology. @todo: complete methods.. """ - def __init__(self, uri, rdftype=None, namespaces=None, ext_model=False): + def __init__(self, uri, rdftype=None, namespaces=None, ext_model=False, pref_title="qname", pref_lang="en"): """ ... """ - super(OntoSKOSConcept, self).__init__(uri, rdftype, namespaces, - ext_model) + super().__init__(uri, rdftype, namespaces, + ext_model, pref_title=pref_title, pref_lang=pref_lang) self.slug = "concept-" + slugify(self.qname) self.instance_of = [] self.ontology = None @@ -436,17 +472,17 @@ def describe(self): self.printGenericTree() -class OntoShape(RDF_Entity): +class OntoShape(RdfEntity): """ Python representation of a SHACL shape. """ - def __init__(self, uri, rdftype=None, namespaces=None, ext_model=False): + def __init__(self, uri, rdftype=None, namespaces=None, ext_model=False, pref_title="qname", pref_lang="en"): """ ... """ - super(OntoShape, self).__init__(uri, rdftype, namespaces, ext_model) + super().__init__(uri, rdftype, namespaces, ext_model, pref_title=pref_title, pref_lang=pref_lang) self.slug = "shape-" + slugify(self.qname) self.ontology = None self.targetClasses = [] diff --git a/ontospy/core/ontospy.py b/ontospy/core/ontospy.py index 105db0c6..3c184134 100755 --- a/ontospy/core/ontospy.py +++ b/ontospy/core/ontospy.py @@ -8,18 +8,8 @@ """ from __future__ import print_function - -import sys -import os -import time -import optparse from itertools import chain -try: - import urllib2 -except ImportError: - import urllib.request as urllib2 - import rdflib from .utils import * @@ -29,1097 +19,1178 @@ class Ontospy(object): - """ - Object that extracts schema definitions (aka 'ontologies') from an rdf graph. - - In [3]: import ontospy - - In [5]: o = ontospy.Ontospy() - - In [7]: o.load_rdf("foaf.rdf") - - In [11]: o.build_all() - - In [13]: o.stats() - Out[13]: - [('Ontologies', 1), - ('Triples', 630), - ('Classes', 15), - ('Properties', 67), - ('Annotation Properties', 7), - ('Object Properties', 34), - ('Datatype Properties', 26), - ('Skos Concepts', 0), - ('Data Sources', 1)] - - """ - - def __init__(self, - uri_or_path=None, - data=None, - file_obj=None, - rdf_format="", - verbose=False, - hide_base_schemas=True, - hide_implicit_types=True, - hide_implicit_preds=True, - sparql_endpoint=None, - credentials=None, - build_all=True): - """ - Load the graph in memory, then setup all necessary attributes. - """ - super(Ontospy, self).__init__() - - self.rdflib_graph = None - self.sparql_endpoint = None - self.credentials = None # tuple: auth credentials for endpoint if needed - self.sources = None - self.sparqlHelper = None - self.namespaces = [] - # entities buckets start with 'all_' - self.all_ontologies = [] - self.all_classes = [] - self.all_properties = [] - self.all_properties_annotation = [] - self.all_properties_object = [] - self.all_properties_datatype = [] - self.all_skos_concepts = [] - self.all_shapes = [] - # self.all_individuals = [] - self.toplayer_classes = [] - self.toplayer_properties = [] - self.toplayer_skos = [] - self.toplayer_shapes = [] - self.OWLTHING = OntoClass(rdflib.OWL.Thing, rdflib.OWL.Class, self.namespaces) - - # finally: - if uri_or_path or data or file_obj: - self.load_rdf(uri_or_path, data, file_obj, rdf_format, verbose, - hide_base_schemas, hide_implicit_types, - hide_implicit_preds) - if build_all: - self.build_all( - verbose=verbose, - hide_base_schemas=hide_base_schemas, - hide_implicit_types=hide_implicit_types, - hide_implicit_preds=hide_implicit_preds) - elif sparql_endpoint: # by default entities are not extracted - self.load_sparql(sparql_endpoint, verbose, hide_base_schemas, - hide_implicit_types, hide_implicit_preds, credentials) - else: - pass - - def __repr__(self): - """ - Return some info for the ontospy instance. - - note: if it's a sparql backend, limit the info returned to avoid long queries (tip: a statement like `if self.rdflib_graph` on a sparql endpoint is enough to cause a long query!) - - """ - if self.sparql_endpoint and self.rdflib_graph != None: - return ")>" % self.sparql_endpoint - elif self.rdflib_graph != None: - return "" % (len(self.rdflib_graph)) - else: - return "" - - def load_rdf(self, - uri_or_path=None, - data=None, - file_obj=None, - rdf_format="", - verbose=False, - hide_base_schemas=True, - hide_implicit_types=True, - hide_implicit_preds=True): - """Load an RDF source into an ontospy/rdflib graph""" - loader = RDFLoader(verbose=verbose) - loader.load(uri_or_path, data, file_obj, rdf_format) - self.rdflib_graph = loader.rdflib_graph - self.sources = loader.sources_valid - self.sparqlHelper = SparqlHelper(self.rdflib_graph) - self.namespaces = sorted(self.rdflib_graph.namespaces()) - - def load_sparql(self, - sparql_endpoint, - verbose=False, - hide_base_schemas=True, - hide_implicit_types=True, - hide_implicit_preds=True, credentials=None): - """ - Set up a SPARQLStore backend as a virtual ontospy graph - - Note: we're using a 'SPARQLUpdateStore' backend instead of 'SPARQLStore' cause otherwise authentication fails (https://github.com/RDFLib/rdflib/issues/755) - - @TODO this error seems to be fixed in upcoming rdflib versions - https://github.com/RDFLib/rdflib/pull/744 - - """ - try: - # graph = rdflib.Graph('SPARQLStore') - # graph = rdflib.ConjunctiveGraph('SPARQLStore') - graph = rdflib.ConjunctiveGraph('SPARQLUpdateStore') - - if credentials and type(credentials) == tuple: - # https://github.com/RDFLib/rdflib/issues/343 - graph.store.setCredentials(credentials[0], credentials[1]) - # graph.store.setHTTPAuth('BASIC') # graph.store.setHTTPAuth('DIGEST') - - graph.open(sparql_endpoint) - self.rdflib_graph = graph - self.sparql_endpoint = sparql_endpoint - self.sources = [sparql_endpoint] - self.sparqlHelper = SparqlHelper(self.rdflib_graph, self.sparql_endpoint) - self.namespaces = sorted(self.rdflib_graph.namespaces()) - except: - printDebug("Error trying to connect to Endpoint.") - raise - # don't extract entities by default.. - - - # ------------ - # === methods to build python objects === # - # ------------ - - def build_all(self, - verbose=False, - hide_base_schemas=True, - hide_implicit_types=True, - hide_implicit_preds=True): - """ - Extract all ontology entities from an RDF graph and construct Python representations of them. - """ - if verbose: - printDebug("Scanning entities...", "green") - printDebug("----------", "comment") - - self.build_ontologies() - if verbose: - printDebug("Ontologies.........: %d" % len(self.all_ontologies), "comment") - - self.build_classes(hide_base_schemas, hide_implicit_types) - if verbose: - printDebug("Classes............: %d" % len(self.all_classes), "comment") - - self.build_properties(hide_implicit_preds) - if verbose: - printDebug("Properties.........: %d" % len(self.all_properties), "comment") - if verbose: - printDebug("..annotation.......: %d" % len(self.all_properties_annotation), "comment") - if verbose: - printDebug("..datatype.........: %d" % len(self.all_properties_datatype), "comment") - if verbose: - printDebug("..object...........: %d" % len(self.all_properties_object), "comment") - - self.build_skos_concepts() - if verbose: - printDebug("Concepts (SKOS)....: %d" % len(self.all_skos_concepts), "comment") - - self.build_shapes() - if verbose: - printDebug("Shapes (SHACL).....: %d" % len(self.all_shapes), "comment") - - # self.__computeTopLayer() - - self.__computeInferredProperties() - - if verbose: - printDebug("----------", "comment") - - def build_ontologies(self, exclude_BNodes=False, return_string=False): - """ - Extract ontology instances info from the graph, then creates python objects for them. - - Note: often ontology info is nested in structures like this: - - [ a owl:Ontology ; - vann:preferredNamespacePrefix "bsym" ; - vann:preferredNamespaceUri "http://bsym.bloomberg.com/sym/" ] - - Hence there is some logic to deal with these edge cases. - """ - out = [] - - qres = self.sparqlHelper.getOntology() - - if qres: - # NOTE: SPARQL returns a list of rdflib.query.ResultRow (~ tuples..) - - for candidate in qres: - if isBlankNode(candidate[0]): - if exclude_BNodes: - continue - else: - checkDC_ID = [x for x in self.rdflib_graph.objects( - candidate[0], rdflib.namespace.DC.identifier)] - if checkDC_ID: - out += [Ontology(checkDC_ID[0], namespaces=self.namespaces), ] - else: - vannprop = rdflib.URIRef( - "http://purl.org/vocab/vann/preferredNamespaceUri") - vannpref = rdflib.URIRef( - "http://purl.org/vocab/vann/preferredNamespacePrefix") - checkDC_ID = [x for x in self.rdflib_graph.objects( - candidate[0], vannprop)] - if checkDC_ID: - checkDC_prefix = [ - x for x in self.rdflib_graph.objects(candidate[0], vannpref)] - if checkDC_prefix: - out += [Ontology(checkDC_ID[0], - namespaces=self.namespaces, - prefPrefix=checkDC_prefix[0])] - else: - out += [Ontology(checkDC_ID[0], namespaces=self.namespaces)] - - else: - out += [Ontology(candidate[0], namespaces=self.namespaces)] - - else: - pass - # printDebug("No owl:Ontologies found") - - # finally... add all annotations/triples - self.all_ontologies = out - for onto in self.all_ontologies: - onto.triples = self.sparqlHelper.entityTriples(onto.uri) - onto._buildGraph() # force construction of mini graph - - # - # RDFS:class vs OWL:class cf. http://www.w3.org/TR/owl-ref/ section 3.1 - # - - def build_classes(self, hide_base_schemas=True, hide_implicit_types=True): - """ - 2015-06-04: removed sparql 1.1 queries - 2015-05-25: optimized via sparql queries in order to remove BNodes - 2015-05-09: new attempt - - Note: sparqlHelper.getAllClasses() returns a list of tuples, - (class, classRDFtype) - so in some cases there are duplicates if a class is both RDFS.CLass and OWL.Class - In this case we keep only OWL.Class as it is more informative. - """ - - self.all_classes = [] # @todo: keep adding? - - qres = self.sparqlHelper.getAllClasses(hide_base_schemas, - hide_implicit_types) - # print("rdflib query done") - - for class_tuple in qres: - - _uri = class_tuple[0] - try: - _type = class_tuple[1] - except: - _type = "" - - test_existing_cl = self.get_class(uri=_uri) - if not test_existing_cl: - # create it - ontoclass = OntoClass(_uri, _type, self.namespaces) - self.all_classes += [ontoclass] - else: - # if OWL.Class over RDFS.Class - update it - if _type == rdflib.OWL.Class: - test_existing_cl.rdftype = rdflib.OWL.Class - - # print("classes created") - - # add more data - for aClass in self.all_classes: - # print("enriching class", aClass) - aClass.triples = self.sparqlHelper.entityTriples(aClass.uri) - aClass._buildGraph() # force construction of mini graph - - aClass.sparqlHelper = self.sparqlHelper - - # attach to an ontology - for uri in aClass.getValuesForProperty(rdflib.RDFS.isDefinedBy): - onto = self.get_ontology(uri=str(uri)) - if onto: - onto.all_classes += [aClass] - aClass.ontology = onto - - # add direct Supers - directSupers = self.sparqlHelper.getClassDirectSupers(aClass.uri) - - for x in directSupers: - superclass = self.get_class(uri=x[0]) - # note: extra condition to avoid recursive structures - if superclass and superclass.uri != aClass.uri: - aClass._parents.append(superclass) - - # add inverse relationships (= direct subs for superclass) - if aClass not in superclass.children(): - superclass._children.append(aClass) - - # sort alphabetically - self.all_classes = sorted(self.all_classes, key=lambda x: x.qname) - - # compute top layer - # print("calc toplayer") - exit = [] - for c in self.all_classes: - if not c.parents(): - exit += [c] - self.toplayer_classes = exit # sorted(exit, key=lambda x: x.id) # doesnt work - - def build_properties(self, hide_implicit_preds=True): - """ - 2015-06-04: removed sparql 1.1 queries - 2015-06-03: analogous to get classes - - # instantiate properties making sure duplicates are pruned - # but the most specific rdftype is kept - # eg OWL:ObjectProperty over RDF:property - - """ - self.all_properties = [] # @todo: keep adding? - self.all_properties_annotation = [] - self.all_properties_object = [] - self.all_properties_datatype = [] - - qres = self.sparqlHelper.getAllProperties(hide_implicit_preds) - # print("rdflib query done") - - for candidate in qres: - - test_existing_prop = self.get_property(uri=candidate[0]) - if not test_existing_prop: - # create it - self.all_properties += [OntoProperty(candidate[0], candidate[1], self.namespaces)] - else: - # update it - if candidate[1] and (test_existing_prop.rdftype == rdflib.RDF.Property): - test_existing_prop.rdftype = inferMainPropertyType(candidate[1]) - # print("properties created") - - # add more data - for aProp in self.all_properties: - # print("enriching prop..", aProp) - if aProp.rdftype == rdflib.OWL.DatatypeProperty: - self.all_properties_datatype += [aProp] - elif aProp.rdftype == rdflib.OWL.AnnotationProperty: - self.all_properties_annotation += [aProp] - elif aProp.rdftype == rdflib.OWL.ObjectProperty: - self.all_properties_object += [aProp] - else: - pass - - aProp.triples = self.sparqlHelper.entityTriples(aProp.uri) - aProp._buildGraph() # force construction of mini graph - - # attach to an ontology [2015-06-15: no property type distinction yet] - for uri in aProp.getValuesForProperty(rdflib.RDFS.isDefinedBy): - onto = self.get_ontology(uri=str(uri)) - if onto: - onto.all_properties += [aProp] - aProp.ontology = onto - - self.__buildDomainRanges(aProp) - - # add direct Supers - directSupers = self.sparqlHelper.getPropDirectSupers(aProp.uri) - - for x in directSupers: - superprop = self.get_property(uri=x[0]) - # note: extra condition to avoid recursive structures - if superprop and superprop.uri != aProp.uri: - aProp._parents.append(superprop) - - # add inverse relationships (= direct subs for superprop) - if aProp not in superprop.children(): - superprop._children.append(aProp) - - # sort alphabetically - self.all_properties = sorted(self.all_properties, key=lambda x: x.qname) - - # computer top layer for properties - # print("calc toplayer") - exit = [] - for c in self.all_properties: - if not c.parents(): - exit += [c] - self.toplayer_properties = exit # sorted(exit, key=lambda x: x.id) # doesnt work - - def build_skos_concepts(self): - """ - 2015-08-19: first draft - """ - self.all_skos_concepts = [] # @todo: keep adding? - - qres = self.sparqlHelper.getSKOSInstances() - # print("rdflib query done") - - for candidate in qres: - - test_existing_cl = self.get_skos(uri=candidate[0]) - if not test_existing_cl: - # create it - self.all_skos_concepts += [OntoSKOSConcept(candidate[0], None, self.namespaces)] - else: - pass - # print("concepts created") - # add more data - skos = rdflib.Namespace('http://www.w3.org/2004/02/skos/core#') - - for aConcept in self.all_skos_concepts: - # print("enriching concept...", aConcept) - aConcept.rdftype = skos['Concept'] - aConcept.triples = self.sparqlHelper.entityTriples(aConcept.uri) - aConcept._buildGraph() # force construction of mini graph - - aConcept.sparqlHelper = self.sparqlHelper - - # attach to an ontology - for uri in aConcept.getValuesForProperty(rdflib.RDFS.isDefinedBy): - onto = self.get_ontology(uri=str(uri)) - if onto: - onto.all_skos_concepts += [aConcept] - aConcept.ontology = onto - - # add direct Supers - directSupers = self.sparqlHelper.getSKOSDirectSupers(aConcept.uri) - - for x in directSupers: - superclass = self.get_skos(uri=x[0]) - # note: extra condition to avoid recursive structures - if superclass and superclass.uri != aConcept.uri: - aConcept._parents.append(superclass) - - # add inverse relationships (= direct subs for superclass) - if aConcept not in superclass.children(): - superclass._children.append(aConcept) - - # sort alphabetically - self.all_skos_concepts = sorted(self.all_skos_concepts, key=lambda x: x.qname) - - # compute top layer for skos - exit = [] - for c in self.all_skos_concepts: - if not c.parents(): - exit += [c] - self.toplayer_skos = exit # sorted(exit, key=lambda x: x.id) # doesnt work - - def build_shapes(self): - """ - Extract SHACL data shapes from the rdf graph. - - - Instatiate the Shape Python objects and relate it to existing classes, - if available. - """ - self.all_shapes = [] # @todo: keep adding? - - qres = self.sparqlHelper.getShapes() - - for candidate in qres: - - test_existing_cl = self.get_any_entity(uri=candidate[0]) - if not test_existing_cl: - # create it - self.all_shapes += [OntoShape(candidate[0], None, self.namespaces)] - else: - pass - - # add more data - shacl = rdflib.Namespace('http://www.w3.org/ns/shacl#') - - for aShape in self.all_shapes: - - aShape.rdftype = shacl['Shape'] - aShape.triples = self.sparqlHelper.entityTriples(aShape.uri) - aShape._buildGraph() # force construction of mini graph - - aShape.sparqlHelper = self.sparqlHelper - - # attach to a class - for uri in aShape.getValuesForProperty(shacl['targetClass']): - aclass = self.get_class(str(uri)) - if aclass: - aShape.targetClasses += [aclass] - aclass.all_shapes += [aShape] - for propertyUri in aShape.getValuesForProperty(shacl['path']): #add shaped properties of this class. later can be used for ontodocs - propType = self.get_property(str(propertyUri)) - if propType: - aclass.shapedProperties += [{'shape': aShape, 'property': propType}] - - - - # sort alphabetically - self.all_shapes = sorted(self.all_shapes, key=lambda x: x.qname) - - # compute top layer - exit = [] - for c in self.all_shapes: - if not c.parents(): - exit += [c] - self.toplayer_shapes = exit # sorted(exit, key=lambda x: x.id) # doesnt work - - def build_entity_from_uri(self, uri, ontospyClass=None): - """ - Extract RDF statements having a URI as subject, then instantiate the RDF_Entity Python object so that it can be queried further. - - Passing allows to instantiate a user-defined RDF_Entity subclass. - - NOTE: the entity is not attached to any index. In future version we may create an index for these (individuals?) keeping into account that any existing model entity could be (re)created this way. - """ - if not ontospyClass: - ontospyClass = RDF_Entity - elif not issubclass(ontospyClass, RDF_Entity): - click.secho("Error: <%s> is not a subclass of ontospy.RDF_Entity" % str(ontospyClass)) - return None - else: - pass - qres = self.sparqlHelper.entityTriples(uri) - if qres: - entity = ontospyClass(rdflib.URIRef(uri), None, self.namespaces) - entity.triples = qres - entity._buildGraph() # force construction of mini graph - # try to add class info - test = entity.getValuesForProperty(rdflib.RDF.type) - if test: - entity.rdftype = test - entity.rdftype_qname = [entity._build_qname(x) for x in test] - return entity - else: - return None - - # ------------ - # === methods to refine the ontology structure === # - # ------------ - - def __buildDomainRanges(self, aProp): - """ - extract domain/range details and add to Python objects - """ - - domains = chain(aProp.rdflib_graph.objects( - None, rdflib.term.URIRef(u'http://schema.org/domainIncludes')), aProp.rdflib_graph.objects( - None, rdflib.RDFS.domain)) - - ranges = chain(aProp.rdflib_graph.objects( - None, rdflib.term.URIRef(u'http://schema.org/rangeIncludes')), aProp.rdflib_graph.objects( - None, rdflib.RDFS.range)) - - for x in domains: - if isBlankNode(x): - aProp.domains += [RDF_Entity(x, None, self.namespaces, is_Bnode=True)] - else: - aClass = self.get_class(uri=str(x)) - if aClass: - aProp.domains += [aClass] - aClass.domain_of += [aProp] - else: - # edge case: it's not an OntoClass instance - aProp.domains += [OntoClass(x, None, self.namespaces, ext_model=True)] - - for x in ranges: - if isBlankNode(x): - aProp.domains += [RDF_Entity(x, None, self.namespaces, is_Bnode=True)] - else: - aClass = self.get_class(uri=str(x)) - if aClass: - aProp.ranges += [aClass] - aClass.range_of += [aProp] - else: - # eg a DataType property has xsd:STRING - # here we're storing an ontospy entities but not adding it to - # the main index - aProp.ranges += [OntoClass(x, None, self.namespaces, ext_model=True)] - - def __computeTopLayer(self): - """ - deprecated: now this is calculated when entities get extracted - """ - - exit = [] - for c in self.all_classes: - if not c.parents(): - exit += [c] - self.toplayer_classes = exit # sorted(exit, key=lambda x: x.id) # doesnt work - - # properties - exit = [] - for c in self.all_properties: - if not c.parents(): - exit += [c] - self.toplayer_properties = exit # sorted(exit, key=lambda x: x.id) # doesnt work - - # skos - exit = [] - for c in self.all_skos_concepts: - if not c.parents(): - exit += [c] - self.toplayer_skos = exit # sorted(exit, key=lambda x: x.id) # doesnt work - - def __computeInferredProperties(self): - """ - - :return: attach a list of dicts to each class, detailing valid props up the subsumption tree - """ - exit = [] - for c in self.all_classes: - c.domain_of_inferred = self.getInferredPropertiesForClass(c, "domain_of") - c.range_of_inferred = self.getInferredPropertiesForClass(c, "range_of") - - def getInferredPropertiesForClass(self, aClass, rel="domain_of"): - """ - returns all properties valid for a class (as they have it in their domain) - recursively ie traveling up the descendants tree - Note: results in a list of dicts including itself - Note [2]: all properties with no domain info are added at the top as [None, props] - - :return: - [{: - [,, - etc....]}, - {: - [, etc...]}, - ] - """ - _list = [] - - if rel == "domain_of": - _list.append({aClass: aClass.domain_of}) - for x in aClass.ancestors(): - if x.domain_of: - _list.append({x: x.domain_of}) - - # add properties from Owl:Thing ie the inference layer - - topLevelProps = [p for p in self.all_properties if p.domains == []] - if topLevelProps: - _list.append({self.OWLTHING: topLevelProps}) - - elif rel == "range_of": - _list.append({aClass: aClass.range_of}) - for x in aClass.ancestors(): - if x.domain_of: - _list.append({x: x.range_of}) - - # add properties from Owl:Thing ie the inference layer - - topLevelProps = [p for p in self.all_properties if p.ranges == []] - if topLevelProps: - _list.append({self.OWLTHING: topLevelProps}) - - return _list - - - - # =============== - # methods for retrieving objects - # ================ - - def get_class(self, id=None, uri=None, match=None): - """ - get the saved-class with given ID or via other methods... - - Note: it tries to guess what is being passed.. - - In [1]: g.get_class(uri='http://www.w3.org/2000/01/rdf-schema#Resource') - Out[1]: - - In [2]: g.get_class(10) - Out[2]: - - In [3]: g.get_class(match="person") - Out[3]: - [, - , - ] - - """ - - if not id and not uri and not match: - return None - - if type(id) == type("string"): - uri = id - id = None - if not is_http(uri): - match = uri - uri = None - if match: - if type(match) != type("string"): - return [] - res = [] - if ":" in match: # qname - for x in self.all_classes: - if match.lower() in x.qname.lower(): - res += [x] - else: - for x in self.all_classes: - if match.lower() in x.uri.lower(): - res += [x] - return res - else: - for x in self.all_classes: - if id and x.id == id: - return x - if uri and x.uri.lower() == uri.lower(): - return x - return None - - def get_property(self, id=None, uri=None, match=None): - """ - get the saved-class with given ID or via other methods... - - Note: analogous to getClass method - """ - - if not id and not uri and not match: - return None - - if type(id) == type("string"): - uri = id - id = None - if not is_http(uri): - match = uri - uri = None - if match: - if type(match) != type("string"): - return [] - res = [] - if ":" in match: # qname - for x in self.all_properties: - if match.lower() in x.qname.lower(): - res += [x] - else: - for x in self.all_properties: - if match.lower() in x.uri.lower(): - res += [x] - return res - else: - for x in self.all_properties: - if id and x.id == id: - return x - if uri and x.uri.lower() == uri.lower(): - return x - return None - - def get_skos(self, id=None, uri=None, match=None): - """ - get the saved skos concept with given ID or via other methods... - - Note: it tries to guess what is being passed as above - """ - - if not id and not uri and not match: - return None - - if type(id) == type("string"): - uri = id - id = None - if not is_http(uri): - match = uri - uri = None - if match: - if type(match) != type("string"): - return [] - res = [] - if ":" in match: # qname - for x in self.all_skos_concepts: - if match.lower() in x.qname.lower(): - res += [x] - else: - for x in self.all_skos_concepts: - if match.lower() in x.uri.lower(): - res += [x] - return res - else: - for x in self.all_skos_concepts: - if id and x.id == id: - return x - if uri and x.uri.lower() == uri.lower(): - return x - return None - - def get_any_entity(self, id=None, uri=None, match=None): - """ - get a generic entity with given ID or via other methods... - """ - - if not id and not uri and not match: - return None - - if type(id) == type("string"): - uri = id - id = None - if not is_http(uri): - match = uri - uri = None - if match: - if type(match) != type("string"): - return [] - res = [] - if ":" in match: # qname - for x in self.all_classes: - if match.lower() in x.qname.lower(): - res += [x] - for x in self.all_properties: - if match.lower() in x.qname.lower(): - res += [x] - else: - for x in self.all_classes: - if match.lower() in x.uri.lower(): - res += [x] - for x in self.all_properties: - if match.lower() in x.uri.lower(): - res += [x] - return res - else: - for x in self.all_classes: - if id and x.id == id: - return x - if uri and x.uri.lower() == uri.lower(): - return x - for x in self.all_properties: - if id and x.id == id: - return x - if uri and x.uri.lower() == uri.lower(): - return x - return None - - def get_ontology(self, id=None, uri=None, match=None): - """ - get the saved-ontology with given ID or via other methods... - """ - - if not id and not uri and not match: - return None - - if type(id) == type("string"): - uri = id - id = None - if not is_http(uri): - match = uri - uri = None - if match: - if type(match) != type("string"): - return [] - res = [] - for x in self.all_ontologies: - if match.lower() in x.uri.lower(): - res += [x] - return res - else: - for x in self.all_ontologies: - if id and x.id == id: - return x - if uri and x.uri.lower() == uri.lower(): - return x - return None - - def nextClass(self, classuri): - """Returns the next class in the list of classes. If it's the last one, returns the first one.""" - if classuri == self.all_classes[-1].uri: - return self.all_classes[0] - flag = False - for x in self.all_classes: - if flag == True: - return x - if x.uri == classuri: - flag = True - return None - - def nextProperty(self, propuri): - """Returns the next property in the list of properties. If it's the last one, returns the first one.""" - if propuri == self.all_properties[-1].uri: - return self.all_properties[0] - flag = False - for x in self.all_properties: - if flag == True: - return x - if x.uri == propuri: - flag = True - return None - - def nextConcept(self, concepturi): - """Returns the next skos concept in the list of concepts. If it's the last one, returns the first one.""" - if concepturi == self.all_skos_concepts[-1].uri: - return self.all_skos_concepts[0] - flag = False - for x in self.all_skos_concepts: - if flag == True: - return x - if x.uri == concepturi: - flag = True - return None - - def ontologyClassTree(self): - """ - Returns a dict representing the ontology tree - Top level = {0:[top classes]} - Multi inheritance is represented explicitly - """ - treedict = {} - if self.all_classes: - treedict[0] = self.toplayer_classes - for element in self.all_classes: - if element.children(): - treedict[element] = element.children() - return treedict - return treedict - - def ontologyPropTree(self): - """ - Returns a dict representing the ontology tree - Top level = {0:[top properties]} - Multi inheritance is represented explicitly - """ - treedict = {} - if self.all_properties: - treedict[0] = self.toplayer_properties - for element in self.all_properties: - if element.children(): - treedict[element] = element.children() - return treedict - return treedict - - def ontologyConceptTree(self): - """ - Returns a dict representing the skos tree - Top level = {0:[top concepts]} - Multi inheritance is represented explicitly - """ - treedict = {} - if self.all_skos_concepts: - treedict[0] = self.toplayer_skos - for element in self.all_skos_concepts: - if element.children(): - treedict[element] = element.children() - return treedict - return treedict - - def ontologyShapeTree(self): - """ - Returns a dict representing the ontology tree - Top level = {0:[top properties]} - Multi inheritance is represented explicitly - """ - treedict = {} - if self.all_shapes: - treedict[0] = self.toplayer_shapes - for element in self.all_shapes: - if element.children(): - treedict[element] = element.children() - return treedict - return treedict - - - # ------------ - # === utils === # - # ------------ - - - def rdf_source(self, format="turtle"): - """ - Wrapper for rdflib serializer method. - Valid options are: xml, n3, turtle, nt, pretty-xml, json-ld [trix not working out of the box] - """ - s = self.rdflib_graph.serialize(format=format) - if isinstance(s, bytes): - s = s.decode('utf-8') - return s - - - def serialize(self, format="turtle"): - "for backward compatibility" - return self.rdf_source(format) - - def query(self, stringa): - """SPARQL query / wrapper for rdflib sparql query method """ - qres = self.rdflib_graph.query(stringa) - return list(qres) - def sparql(self, stringa): - "SPARQL query / replacement for query" - return self.query(stringa) - - def stats(self): - """ shotcut to pull out useful info for a graph""" - out = [] - out += [("Ontologies", len(self.all_ontologies))] - out += [("Triples", self.triplesCount())] - out += [("Classes", len(self.all_classes))] - out += [("Properties", len(self.all_properties))] - out += [("Annotation Properties", len(self.all_properties_annotation))] - out += [("Object Properties", len(self.all_properties_object))] - out += [("Datatype Properties", len(self.all_properties_datatype))] - out += [("Skos Concepts", len(self.all_skos_concepts))] - out += [("Data Shapes", len(self.all_shapes))] - # out += [("Individuals", len(self.individuals))] @TODO - out += [("Data Sources", len(self.sources))] - return out - - def triplesCount(self): - """ - - 2016-08-18 the try/except is a dirty solution to a problem - emerging with counting graph length on cached Graph objects.. - """ - # @todo investigate what's going on.. - # click.secho(unicode(type(self.rdflib_graph)), fg="red") - try: - return len(self.rdflib_graph) - except: - click.secho("Ontospy: error counting graph length..", fg="red") - return 0 - - - def printClassTree(self, element=None, showids=False, labels=False, showtype=False): - """ - Print nicely into stdout the class tree of an ontology - - Note: indentation is made so that ids up to 3 digits fit in, plus a space. - [123]1-- - [1]123-- - [12]12-- - """ - TYPE_MARGIN = 11 # length for owl:class etc.. - - if not element: # first time - for x in self.toplayer_classes: - printGenericTree(x, 0, showids, labels, showtype, TYPE_MARGIN) - - else: - printGenericTree(element, 0, showids, labels, showtype, TYPE_MARGIN) - - def printPropertyTree(self, element=None, showids=False, labels=False, showtype=False): - """ - Print nicely into stdout the property tree of an ontology - - Note: indentation is made so that ids up to 3 digits fit in, plus a space. - [123]1-- - [1]123-- - [12]12-- - """ - TYPE_MARGIN = 18 # length for owl:AnnotationProperty etc.. - - if not element: # first time - for x in self.toplayer_properties: - printGenericTree(x, 0, showids, labels, showtype, TYPE_MARGIN) - - else: - printGenericTree(element, 0, showids, labels, showtype, TYPE_MARGIN) - - def printSkosTree(self, element=None, showids=False, labels=False, showtype=False): - """ - Print nicely into stdout the SKOS tree of an ontology - - Note: indentation is made so that ids up to 3 digits fit in, plus a space. - [123]1-- - [1]123-- - [12]12-- - """ - TYPE_MARGIN = 13 # length for skos:concept - - if not element: # first time - for x in self.toplayer_skos: - printGenericTree(x, 0, showids, labels, showtype, TYPE_MARGIN) - - else: - printGenericTree(element, 0, showids, labels, showtype, TYPE_MARGIN) + """ + Object that extracts schema definitions (aka 'ontologies') from an rdf graph. + + In [3]: import ontospy + + In [5]: o = ontospy.Ontospy() + + In [7]: o.load_rdf("foaf.rdf") + + In [11]: o.build_all() + + In [13]: o.stats() + Out[13]: + [('Ontologies', 1), + ('Triples', 630), + ('Classes', 15), + ('Properties', 67), + ('Annotation Properties', 7), + ('Object Properties', 34), + ('Datatype Properties', 26), + ('Skos Concepts', 0), + ('Data Sources', 1)] + + """ + + def __init__(self, + uri_or_path=None, + data=None, + file_obj=None, + rdf_format="", + verbose=False, + hide_base_schemas=True, + hide_implicit_types=True, + hide_implicit_preds=True, + sparql_endpoint=None, + credentials=None, + build_all=True, + pref_title="qname", + pref_lang="en", + ): + """Load the graph in memory, then setup all necessary attributes. + + Parameters + ---------- + uri_or_path : [type], optional + [description], by default None + data : [type], optional + [description], by default None + file_obj : [type], optional + [description], by default None + rdf_format : str, optional + [description], by default "" + verbose : bool, optional + [description], by default False + hide_base_schemas : bool, optional + [description], by default True + hide_implicit_types : bool, optional + [description], by default True + hide_implicit_preds : bool, optional + [description], by default True + sparql_endpoint : [type], optional + [description], by default None + credentials : [type], optional + [description], by default None + build_all : bool, optional + [description], by default True + pref_title : str, optional + How to display entities by default. Two options accepted: "qname" (default) or "label" for rdfs:label. + pref_lang : str, optional + Default: 'en' + """ + + super(Ontospy, self).__init__() + + self.rdflib_graph = None + self.sparql_endpoint = None + self.credentials = None # tuple: auth credentials for endpoint if needed + self.sources = None + self.sparqlHelper = None + self.pref_title = pref_title + self.pref_lang = pref_lang + self.namespaces = [] + # entities buckets start with 'all_' + self.all_ontologies = [] + self.all_classes = [] + self.all_properties = [] + self.all_properties_annotation = [] + self.all_properties_object = [] + self.all_properties_datatype = [] + self.all_skos_concepts = [] + self.all_shapes = [] + # self.all_individuals = [] + self.toplayer_classes = [] + self.toplayer_properties = [] + self.toplayer_skos = [] + self.toplayer_shapes = [] + self.OWLTHING = OntoClass(rdflib.OWL.Thing, + rdflib.OWL.Class, + self.namespaces, + False, + self.pref_title, + self.pref_lang) + + # finally: + if uri_or_path or data or file_obj: + self.load_rdf(uri_or_path, data, file_obj, rdf_format, verbose) + if build_all: + self.build_all( + verbose=verbose, + hide_base_schemas=hide_base_schemas, + hide_implicit_types=hide_implicit_types, + hide_implicit_preds=hide_implicit_preds, + ) + elif sparql_endpoint: # by default entities are not extracted + self.load_sparql(sparql_endpoint, verbose, credentials) + else: + pass + + def __repr__(self): + """ + Return some info for the ontospy instance. + + note: if it's a sparql backend, limit the info returned to avoid long queries (tip: a statement like `if self.rdflib_graph` on a sparql endpoint is enough to cause a long query!) + + """ + if self.sparql_endpoint and self.rdflib_graph != None: + return ")>" % self.sparql_endpoint + elif self.rdflib_graph != None: + return "" % (len(self.rdflib_graph)) + else: + return "" + + def load_rdf(self, + uri_or_path=None, + data=None, + file_obj=None, + rdf_format="", + verbose=False): + """Load an RDF source into an ontospy/rdflib graph""" + loader = RDFLoader(verbose=verbose) + loader.load(uri_or_path, data, file_obj, rdf_format) + self.rdflib_graph = loader.rdflib_graph + self.sources = loader.sources_valid + self.sparqlHelper = SparqlHelper(self.rdflib_graph) + self.namespaces = sorted(self.rdflib_graph.namespaces()) + + def load_sparql(self, + sparql_endpoint, + verbose=False, + credentials=None): + """ + Set up a SPARQLStore backend as a virtual ontospy graph + + Note: we're using a 'SPARQLUpdateStore' backend instead of 'SPARQLStore' cause otherwise authentication fails (https://github.com/RDFLib/rdflib/issues/755) + + @TODO this error seems to be fixed in upcoming rdflib versions + https://github.com/RDFLib/rdflib/pull/744 + + """ + try: + # graph = rdflib.Graph('SPARQLStore') + # graph = rdflib.ConjunctiveGraph('SPARQLStore') + graph = rdflib.ConjunctiveGraph('SPARQLUpdateStore') + + if credentials and type(credentials) == tuple: + # https://github.com/RDFLib/rdflib/issues/343 + graph.store.setCredentials(credentials[0], credentials[1]) + # graph.store.setHTTPAuth('BASIC') # graph.store.setHTTPAuth('DIGEST') + + graph.open(sparql_endpoint) + self.rdflib_graph = graph + self.sparql_endpoint = sparql_endpoint + self.sources = [sparql_endpoint] + self.sparqlHelper = SparqlHelper(self.rdflib_graph, self.sparql_endpoint) + self.namespaces = sorted(self.rdflib_graph.namespaces()) + except: + printDebug("Error trying to connect to Endpoint.") + raise + # don't extract entities by default.. + + + # ------------ + # === methods to build python objects === # + # ------------ + + def build_all(self, + verbose=False, + hide_base_schemas=True, + hide_implicit_types=True, + hide_implicit_preds=True, + ): + """ + Extract all ontology entities from an RDF graph and construct Python representations of them. + """ + if verbose: + printDebug("Scanning entities...", "green") + printDebug("----------", "comment") + + self.build_ontologies() + if verbose: + printDebug("Ontologies.........: %d" % len(self.all_ontologies), "comment") + + self.build_classes(hide_base_schemas, hide_implicit_types) + if verbose: + printDebug("Classes............: %d" % len(self.all_classes), "comment") + + self.build_properties(hide_implicit_preds) + if verbose: + printDebug("Properties.........: %d" % len(self.all_properties), "comment") + if verbose: + printDebug("..annotation.......: %d" % len(self.all_properties_annotation), "comment") + if verbose: + printDebug("..datatype.........: %d" % len(self.all_properties_datatype), "comment") + if verbose: + printDebug("..object...........: %d" % len(self.all_properties_object), "comment") + + self.build_skos_concepts() + if verbose: + printDebug("Concepts (SKOS)....: %d" % len(self.all_skos_concepts), "comment") + + self.build_shapes() + if verbose: + printDebug("Shapes (SHACL).....: %d" % len(self.all_shapes), "comment") + + # self.__computeTopLayer() + + self.__computeInferredProperties() + + if verbose: + printDebug("----------", "comment") + + def build_ontologies(self, exclude_BNodes=False, return_string=False): + """ + Extract ontology instances info from the graph, then creates python objects for them. + + Note: often ontology info is nested in structures like this: + + [ a owl:Ontology ; + vann:preferredNamespacePrefix "bsym" ; + vann:preferredNamespaceUri "http://bsym.bloomberg.com/sym/" ] + + Hence there is some logic to deal with these edge cases. + """ + out = [] + + qres = self.sparqlHelper.getOntology() + + if qres: + # NOTE: SPARQL returns a list of rdflib.query.ResultRow (~ tuples..) + + for candidate in qres: + if isBlankNode(candidate[0]): + if exclude_BNodes: + continue + else: + checkDC_ID = [x for x in self.rdflib_graph.objects( + candidate[0], rdflib.namespace.DC.identifier)] + if checkDC_ID: + out += [Ontology(checkDC_ID[0], + namespaces=self.namespaces, + pref_title=self.pref_title), + ] + else: + vannprop = rdflib.URIRef( + "http://purl.org/vocab/vann/preferredNamespaceUri") + vannpref = rdflib.URIRef( + "http://purl.org/vocab/vann/preferredNamespacePrefix") + checkDC_ID = [x for x in self.rdflib_graph.objects( + candidate[0], vannprop)] + if checkDC_ID: + checkDC_prefix = [ + x for x in self.rdflib_graph.objects(candidate[0], vannpref)] + if checkDC_prefix: + out += [Ontology(checkDC_ID[0], + namespaces=self.namespaces, + pref_prefix=checkDC_prefix[0], + pref_title=self.pref_title, + pref_lang=self.pref_lang, + ) + ] + else: + out += [Ontology(checkDC_ID[0], + namespaces=self.namespaces, + pref_title=self.pref_title, + pref_lang=self.pref_lang, + )] + + else: + out += [Ontology(candidate[0], + namespaces=self.namespaces, + pref_title=self.pref_title, + pref_lang=self.pref_lang, + )] + + else: + pass + # printDebug("No owl:Ontologies found") + + # finally... add all annotations/triples + self.all_ontologies = out + for onto in self.all_ontologies: + onto.triples = self.sparqlHelper.entityTriples(onto.uri) + onto._buildGraph() # force construction of mini graph + + # + # RDFS:class vs OWL:class cf. http://www.w3.org/TR/owl-ref/ section 3.1 + # + + def build_classes(self, hide_base_schemas=True, hide_implicit_types=True): + """ + 2015-06-04: removed sparql 1.1 queries + 2015-05-25: optimized via sparql queries in order to remove BNodes + 2015-05-09: new attempt + + Note: sparqlHelper.getAllClasses() returns a list of tuples, + (class, classRDFtype) + so in some cases there are duplicates if a class is both RDFS.CLass and OWL.Class + In this case we keep only OWL.Class as it is more informative. + """ + + self.all_classes = [] # @todo: keep adding? + + qres = self.sparqlHelper.getAllClasses(hide_base_schemas, + hide_implicit_types) + # print("rdflib query done") + + for class_tuple in qres: + + _uri = class_tuple[0] + try: + _type = class_tuple[1] + except: + _type = "" + + test_existing_cl = self.get_class(uri=_uri) + if not test_existing_cl: + # create it + ontoclass = OntoClass(_uri, _type, + self.namespaces, + False, + self.pref_title, + self.pref_lang) + self.all_classes += [ontoclass] + else: + # if OWL.Class over RDFS.Class - update it + if _type == rdflib.OWL.Class: + test_existing_cl.rdftype = rdflib.OWL.Class + + # print("classes created") + + # add more data + for aClass in self.all_classes: + # print("enriching class", aClass) + aClass.triples = self.sparqlHelper.entityTriples(aClass.uri) + aClass._buildGraph() # force construction of mini graph + + aClass.sparqlHelper = self.sparqlHelper + + # attach to an ontology + for uri in aClass.getValuesForProperty(rdflib.RDFS.isDefinedBy): + onto = self.get_ontology(uri=str(uri)) + if onto: + onto.all_classes += [aClass] + aClass.ontology = onto + + # add direct Supers + directSupers = self.sparqlHelper.getClassDirectSupers(aClass.uri) + + for x in directSupers: + superclass = self.get_class(uri=x[0]) + # note: extra condition to avoid recursive structures + if superclass and superclass.uri != aClass.uri: + aClass._parents.append(superclass) + + # add inverse relationships (= direct subs for superclass) + if aClass not in superclass.children(): + superclass._children.append(aClass) + + # sort alphabetically + self.all_classes = sorted(self.all_classes, key=lambda x: x.qname) + + # compute top layer + # print("calc toplayer") + exit = [] + for c in self.all_classes: + if not c.parents(): + exit += [c] + self.toplayer_classes = exit # sorted(exit, key=lambda x: x.id) # doesnt work + + def build_properties(self, hide_implicit_preds=True): + """ + 2015-06-04: removed sparql 1.1 queries + 2015-06-03: analogous to get classes + + # instantiate properties making sure duplicates are pruned + # but the most specific rdftype is kept + # eg OWL:ObjectProperty over RDF:property + + """ + self.all_properties = [] # @todo: keep adding? + self.all_properties_annotation = [] + self.all_properties_object = [] + self.all_properties_datatype = [] + + qres = self.sparqlHelper.getAllProperties(hide_implicit_preds) + # print("rdflib query done") + + for candidate in qres: + + test_existing_prop = self.get_property(uri=candidate[0]) + if not test_existing_prop: + # create it + self.all_properties += [OntoProperty(candidate[0], candidate[1], + self.namespaces, + False, + self.pref_title, + self.pref_lang, + )] + else: + # update it + if candidate[1] and (test_existing_prop.rdftype == rdflib.RDF.Property): + test_existing_prop.rdftype = inferMainPropertyType(candidate[1]) + # print("properties created") + + # add more data + for aProp in self.all_properties: + # print("enriching prop..", aProp) + if aProp.rdftype == rdflib.OWL.DatatypeProperty: + self.all_properties_datatype += [aProp] + elif aProp.rdftype == rdflib.OWL.AnnotationProperty: + self.all_properties_annotation += [aProp] + elif aProp.rdftype == rdflib.OWL.ObjectProperty: + self.all_properties_object += [aProp] + else: + pass + + aProp.triples = self.sparqlHelper.entityTriples(aProp.uri) + aProp._buildGraph() # force construction of mini graph + + # attach to an ontology [2015-06-15: no property type distinction yet] + for uri in aProp.getValuesForProperty(rdflib.RDFS.isDefinedBy): + onto = self.get_ontology(uri=str(uri)) + if onto: + onto.all_properties += [aProp] + aProp.ontology = onto + + self.__buildDomainRanges(aProp) + + # add direct Supers + directSupers = self.sparqlHelper.getPropDirectSupers(aProp.uri) + + for x in directSupers: + superprop = self.get_property(uri=x[0]) + # note: extra condition to avoid recursive structures + if superprop and superprop.uri != aProp.uri: + aProp._parents.append(superprop) + + # add inverse relationships (= direct subs for superprop) + if aProp not in superprop.children(): + superprop._children.append(aProp) + + # sort alphabetically + self.all_properties = sorted(self.all_properties, key=lambda x: x.qname) + + # computer top layer for properties + # print("calc toplayer") + exit = [] + for c in self.all_properties: + if not c.parents(): + exit += [c] + self.toplayer_properties = exit # sorted(exit, key=lambda x: x.id) # doesnt work + + def build_skos_concepts(self): + """ + 2015-08-19: first draft + """ + self.all_skos_concepts = [] # @todo: keep adding? + + qres = self.sparqlHelper.getSKOSInstances() + # print("rdflib query done") + + for candidate in qres: + + test_existing_cl = self.get_skos(uri=candidate[0]) + if not test_existing_cl: + # create it + self.all_skos_concepts += [OntoSKOSConcept(candidate[0], None, + self.namespaces, + None, + self.pref_title, + self.pref_lang,) + ] + else: + pass + # print("concepts created") + # add more data + skos = rdflib.Namespace('http://www.w3.org/2004/02/skos/core#') + + for aConcept in self.all_skos_concepts: + # print("enriching concept...", aConcept) + aConcept.rdftype = skos['Concept'] + aConcept.triples = self.sparqlHelper.entityTriples(aConcept.uri) + aConcept._buildGraph() # force construction of mini graph + + aConcept.sparqlHelper = self.sparqlHelper + + # attach to an ontology + for uri in aConcept.getValuesForProperty(rdflib.RDFS.isDefinedBy): + onto = self.get_ontology(uri=str(uri)) + if onto: + onto.all_skos_concepts += [aConcept] + aConcept.ontology = onto + + # add direct Supers + directSupers = self.sparqlHelper.getSKOSDirectSupers(aConcept.uri) + + for x in directSupers: + superclass = self.get_skos(uri=x[0]) + # note: extra condition to avoid recursive structures + if superclass and superclass.uri != aConcept.uri: + aConcept._parents.append(superclass) + + # add inverse relationships (= direct subs for superclass) + if aConcept not in superclass.children(): + superclass._children.append(aConcept) + + # sort alphabetically + self.all_skos_concepts = sorted(self.all_skos_concepts, key=lambda x: x.qname) + + # compute top layer for skos + exit = [] + for c in self.all_skos_concepts: + if not c.parents(): + exit += [c] + self.toplayer_skos = exit # sorted(exit, key=lambda x: x.id) # doesnt work + + def build_shapes(self): + """ + Extract SHACL data shapes from the rdf graph. + + + Instatiate the Shape Python objects and relate it to existing classes, + if available. + """ + self.all_shapes = [] # @todo: keep adding? + + qres = self.sparqlHelper.getShapes() + + for candidate in qres: + + test_existing_cl = self.get_any_entity(uri=candidate[0]) + if not test_existing_cl: + # create it + self.all_shapes += [OntoShape(candidate[0], None, + self.namespaces, + None, + self.pref_title, + self.pref_lang,) + ] + else: + pass + + # add more data + shacl = rdflib.Namespace('http://www.w3.org/ns/shacl#') + + for aShape in self.all_shapes: + + aShape.rdftype = shacl['Shape'] + aShape.triples = self.sparqlHelper.entityTriples(aShape.uri) + aShape._buildGraph() # force construction of mini graph + + aShape.sparqlHelper = self.sparqlHelper + + # attach to a class + for uri in aShape.getValuesForProperty(shacl['targetClass']): + aclass = self.get_class(str(uri)) + if aclass: + aShape.targetClasses += [aclass] + aclass.all_shapes += [aShape] + for propertyUri in aShape.getValuesForProperty(shacl['path']): #add shaped properties of this class. later can be used for ontodocs + propType = self.get_property(str(propertyUri)) + if propType: + aclass.shapedProperties += [{'shape': aShape, 'property': propType}] + + + + # sort alphabetically + self.all_shapes = sorted(self.all_shapes, key=lambda x: x.qname) + + # compute top layer + exit = [] + for c in self.all_shapes: + if not c.parents(): + exit += [c] + self.toplayer_shapes = exit # sorted(exit, key=lambda x: x.id) # doesnt work + + + def build_entity_from_uri(self, uri, ontospyClass=None): + """ + Extract RDF statements having a URI as subject, then instantiate the RdfEntity Python object so that it can be queried further. + + Passing allows to instantiate a user-defined RdfEntity subclass. + + NOTE: the entity is not attached to any index. In future version we may create an index for these (individuals?) keeping into account that any existing model entity could be (re)created this way. + """ + if not ontospyClass: + ontospyClass = RdfEntity + elif not issubclass(ontospyClass, RdfEntity): + click.secho("Error: <%s> is not a subclass of ontospy.RdfEntity" % str(ontospyClass)) + return None + else: + pass + qres = self.sparqlHelper.entityTriples(uri) + if qres: + entity = ontospyClass(rdflib.URIRef(uri), + None, + self.namespaces, + None, + self.pref_title, + self.pref_lang,) + entity.triples = qres + entity._buildGraph() # force construction of mini graph + # try to add class info + test = entity.getValuesForProperty(rdflib.RDF.type) + if test: + entity.rdftype = test + entity.rdftype_qname = [entity._build_qname(x) for x in test] + return entity + else: + return None + + # ------------ + # === methods to refine the ontology structure === # + # ------------ + + def __buildDomainRanges(self, aProp): + """ + extract domain/range details and add to Python objects + """ + + domains = chain(aProp.rdflib_graph.objects( + None, rdflib.term.URIRef(u'http://schema.org/domainIncludes')), aProp.rdflib_graph.objects( + None, rdflib.RDFS.domain)) + + ranges = chain(aProp.rdflib_graph.objects( + None, rdflib.term.URIRef(u'http://schema.org/rangeIncludes')), aProp.rdflib_graph.objects( + None, rdflib.RDFS.range)) + + for x in domains: + if isBlankNode(x): + aProp.domains += [RdfEntity(x, None, self.namespaces, is_Bnode=True)] + else: + aClass = self.get_class(uri=str(x)) + if aClass: + aProp.domains += [aClass] + aClass.domain_of += [aProp] + else: + # edge case: it's not an OntoClass instance + aProp.domains += [OntoClass(x, None, + self.namespaces, + True, # ext_model arg + self.pref_title, + self.pref_lang,)] + + for x in ranges: + if isBlankNode(x): + aProp.domains += [RdfEntity(x, None, self.namespaces, is_Bnode=True)] + else: + aClass = self.get_class(uri=str(x)) + if aClass: + aProp.ranges += [aClass] + aClass.range_of += [aProp] + else: + # eg a DataType property has xsd:STRING + # here we're storing an ontospy entities but not adding it to + # the main index + aProp.ranges += [OntoClass(x, None, + self.namespaces, + True, + self.pref_title, + self.pref_lang,)] + + def __computeTopLayer(self): + """ + deprecated: now this is calculated when entities get extracted + """ + + exit = [] + for c in self.all_classes: + if not c.parents(): + exit += [c] + self.toplayer_classes = exit # sorted(exit, key=lambda x: x.id) # doesnt work + + # properties + exit = [] + for c in self.all_properties: + if not c.parents(): + exit += [c] + self.toplayer_properties = exit # sorted(exit, key=lambda x: x.id) # doesnt work + + # skos + exit = [] + for c in self.all_skos_concepts: + if not c.parents(): + exit += [c] + self.toplayer_skos = exit # sorted(exit, key=lambda x: x.id) # doesnt work + + def __computeInferredProperties(self): + """ + + :return: attach a list of dicts to each class, detailing valid props up the subsumption tree + """ + exit = [] + for c in self.all_classes: + c.domain_of_inferred = self.getInferredPropertiesForClass(c, "domain_of") + c.range_of_inferred = self.getInferredPropertiesForClass(c, "range_of") + + def getInferredPropertiesForClass(self, aClass, rel="domain_of"): + """ + returns all properties valid for a class (as they have it in their domain) + recursively ie traveling up the descendants tree + Note: results in a list of dicts including itself + Note [2]: all properties with no domain info are added at the top as [None, props] + + :return: + [{: + [,, + etc....]}, + {: + [, etc...]}, + ] + """ + _list = [] + + if rel == "domain_of": + _list.append({aClass: aClass.domain_of}) + for x in aClass.ancestors(): + if x.domain_of: + _list.append({x: x.domain_of}) + + # add properties from Owl:Thing ie the inference layer + + topLevelProps = [p for p in self.all_properties if p.domains == []] + if topLevelProps: + _list.append({self.OWLTHING: topLevelProps}) + + elif rel == "range_of": + _list.append({aClass: aClass.range_of}) + for x in aClass.ancestors(): + if x.domain_of: + _list.append({x: x.range_of}) + + # add properties from Owl:Thing ie the inference layer + + topLevelProps = [p for p in self.all_properties if p.ranges == []] + if topLevelProps: + _list.append({self.OWLTHING: topLevelProps}) + + return _list + + + + # =============== + # methods for retrieving objects + # ================ + + def get_class(self, id=None, uri=None, match=None): + """ + get the saved-class with given ID or via other methods... + + Note: it tries to guess what is being passed.. + + In [1]: g.get_class(uri='http://www.w3.org/2000/01/rdf-schema#Resource') + Out[1]: + + In [2]: g.get_class(10) + Out[2]: + + In [3]: g.get_class(match="person") + Out[3]: + [, + , + ] + + """ + + if not id and not uri and not match: + return None + + if type(id) == type("string"): + uri = id + id = None + if not is_http(uri): + match = uri + uri = None + if match: + if type(match) != type("string"): + return [] + res = [] + if ":" in match: # qname + for x in self.all_classes: + if match.lower() in x.qname.lower(): + res += [x] + else: + for x in self.all_classes: + if match.lower() in x.uri.lower(): + res += [x] + return res + else: + for x in self.all_classes: + if id and x.id == id: + return x + if uri and x.uri.lower() == uri.lower(): + return x + return None + + def get_property(self, id=None, uri=None, match=None): + """ + get the saved-class with given ID or via other methods... + + Note: analogous to getClass method + """ + + if not id and not uri and not match: + return None + + if type(id) == type("string"): + uri = id + id = None + if not is_http(uri): + match = uri + uri = None + if match: + if type(match) != type("string"): + return [] + res = [] + if ":" in match: # qname + for x in self.all_properties: + if match.lower() in x.qname.lower(): + res += [x] + else: + for x in self.all_properties: + if match.lower() in x.uri.lower(): + res += [x] + return res + else: + for x in self.all_properties: + if id and x.id == id: + return x + if uri and x.uri.lower() == uri.lower(): + return x + return None + + def get_skos(self, id=None, uri=None, match=None): + """ + get the saved skos concept with given ID or via other methods... + + Note: it tries to guess what is being passed as above + """ + + if not id and not uri and not match: + return None + + if type(id) == type("string"): + uri = id + id = None + if not is_http(uri): + match = uri + uri = None + if match: + if type(match) != type("string"): + return [] + res = [] + if ":" in match: # qname + for x in self.all_skos_concepts: + if match.lower() in x.qname.lower(): + res += [x] + else: + for x in self.all_skos_concepts: + if match.lower() in x.uri.lower(): + res += [x] + return res + else: + for x in self.all_skos_concepts: + if id and x.id == id: + return x + if uri and x.uri.lower() == uri.lower(): + return x + return None + + def get_any_entity(self, id=None, uri=None, match=None): + """ + get a generic entity with given ID or via other methods... + """ + + if not id and not uri and not match: + return None + + if type(id) == type("string"): + uri = id + id = None + if not is_http(uri): + match = uri + uri = None + if match: + if type(match) != type("string"): + return [] + res = [] + if ":" in match: # qname + for x in self.all_classes: + if match.lower() in x.qname.lower(): + res += [x] + for x in self.all_properties: + if match.lower() in x.qname.lower(): + res += [x] + else: + for x in self.all_classes: + if match.lower() in x.uri.lower(): + res += [x] + for x in self.all_properties: + if match.lower() in x.uri.lower(): + res += [x] + return res + else: + for x in self.all_classes: + if id and x.id == id: + return x + if uri and x.uri.lower() == uri.lower(): + return x + for x in self.all_properties: + if id and x.id == id: + return x + if uri and x.uri.lower() == uri.lower(): + return x + return None + + def get_ontology(self, id=None, uri=None, match=None): + """ + get the saved-ontology with given ID or via other methods... + """ + + if not id and not uri and not match: + return None + + if type(id) == type("string"): + uri = id + id = None + if not is_http(uri): + match = uri + uri = None + if match: + if type(match) != type("string"): + return [] + res = [] + for x in self.all_ontologies: + if match.lower() in x.uri.lower(): + res += [x] + return res + else: + for x in self.all_ontologies: + if id and x.id == id: + return x + if uri and x.uri.lower() == uri.lower(): + return x + return None + + def nextClass(self, classuri): + """Returns the next class in the list of classes. If it's the last one, returns the first one.""" + if classuri == self.all_classes[-1].uri: + return self.all_classes[0] + flag = False + for x in self.all_classes: + if flag == True: + return x + if x.uri == classuri: + flag = True + return None + + def nextProperty(self, propuri): + """Returns the next property in the list of properties. If it's the last one, returns the first one.""" + if propuri == self.all_properties[-1].uri: + return self.all_properties[0] + flag = False + for x in self.all_properties: + if flag == True: + return x + if x.uri == propuri: + flag = True + return None + + def nextConcept(self, concepturi): + """Returns the next skos concept in the list of concepts. If it's the last one, returns the first one.""" + if concepturi == self.all_skos_concepts[-1].uri: + return self.all_skos_concepts[0] + flag = False + for x in self.all_skos_concepts: + if flag == True: + return x + if x.uri == concepturi: + flag = True + return None + + def ontologyClassTree(self): + """ + Returns a dict representing the ontology tree + Top level = {0:[top classes]} + Multi inheritance is represented explicitly + """ + treedict = {} + if self.all_classes: + treedict[0] = self.toplayer_classes + for element in self.all_classes: + if element.children(): + treedict[element] = element.children() + return treedict + return treedict + + def ontologyPropTree(self): + """ + Returns a dict representing the ontology tree + Top level = {0:[top properties]} + Multi inheritance is represented explicitly + """ + treedict = {} + if self.all_properties: + treedict[0] = self.toplayer_properties + for element in self.all_properties: + if element.children(): + treedict[element] = element.children() + return treedict + return treedict + + def ontologyConceptTree(self): + """ + Returns a dict representing the skos tree + Top level = {0:[top concepts]} + Multi inheritance is represented explicitly + """ + treedict = {} + if self.all_skos_concepts: + treedict[0] = self.toplayer_skos + for element in self.all_skos_concepts: + if element.children(): + treedict[element] = element.children() + return treedict + return treedict + + def ontologyShapeTree(self): + """ + Returns a dict representing the ontology tree + Top level = {0:[top properties]} + Multi inheritance is represented explicitly + """ + treedict = {} + if self.all_shapes: + treedict[0] = self.toplayer_shapes + for element in self.all_shapes: + if element.children(): + treedict[element] = element.children() + return treedict + return treedict + + + # ------------ + # === utils === # + # ------------ + + + def rdf_source(self, format="turtle"): + """ + Wrapper for rdflib serializer method. + Valid options are: xml, n3, turtle, nt, pretty-xml, json-ld [trix not working out of the box] + """ + s = self.rdflib_graph.serialize(format=format) + if isinstance(s, bytes): + s = s.decode('utf-8') + return s + + + def serialize(self, format="turtle"): + "for backward compatibility" + return self.rdf_source(format) + + def query(self, stringa): + """SPARQL query / wrapper for rdflib sparql query method """ + qres = self.rdflib_graph.query(stringa) + return list(qres) + def sparql(self, stringa): + "SPARQL query / replacement for query" + return self.query(stringa) + + def stats(self): + """ shotcut to pull out useful info for a graph""" + out = [] + out += [("Ontologies", len(self.all_ontologies))] + out += [("Triples", self.triplesCount())] + out += [("Classes", len(self.all_classes))] + out += [("Properties", len(self.all_properties))] + out += [("Annotation Properties", len(self.all_properties_annotation))] + out += [("Object Properties", len(self.all_properties_object))] + out += [("Datatype Properties", len(self.all_properties_datatype))] + out += [("Skos Concepts", len(self.all_skos_concepts))] + out += [("Data Shapes", len(self.all_shapes))] + # out += [("Individuals", len(self.individuals))] @TODO + out += [("Data Sources", len(self.sources))] + return out + + def triplesCount(self): + """ + + 2016-08-18 the try/except is a dirty solution to a problem + emerging with counting graph length on cached Graph objects.. + """ + # @todo investigate what's going on.. + # click.secho(unicode(type(self.rdflib_graph)), fg="red") + try: + return len(self.rdflib_graph) + except: + click.secho("Ontospy: error counting graph length..", fg="red") + return 0 + + + def printClassTree(self, element=None, showids=False, labels=False, showtype=False): + """ + Print nicely into stdout the class tree of an ontology + + Note: indentation is made so that ids up to 3 digits fit in, plus a space. + [123]1-- + [1]123-- + [12]12-- + """ + TYPE_MARGIN = 11 # length for owl:class etc.. + + if not element: # first time + for x in self.toplayer_classes: + printGenericTree(x, 0, showids, labels, showtype, TYPE_MARGIN) + + else: + printGenericTree(element, 0, showids, labels, showtype, TYPE_MARGIN) + + def printPropertyTree(self, element=None, showids=False, labels=False, showtype=False): + """ + Print nicely into stdout the property tree of an ontology + + Note: indentation is made so that ids up to 3 digits fit in, plus a space. + [123]1-- + [1]123-- + [12]12-- + """ + TYPE_MARGIN = 18 # length for owl:AnnotationProperty etc.. + + if not element: # first time + for x in self.toplayer_properties: + printGenericTree(x, 0, showids, labels, showtype, TYPE_MARGIN) + + else: + printGenericTree(element, 0, showids, labels, showtype, TYPE_MARGIN) + + def printSkosTree(self, element=None, showids=False, labels=False, showtype=False): + """ + Print nicely into stdout the SKOS tree of an ontology + + Note: indentation is made so that ids up to 3 digits fit in, plus a space. + [123]1-- + [1]123-- + [12]12-- + """ + TYPE_MARGIN = 13 # length for skos:concept + + if not element: # first time + for x in self.toplayer_skos: + printGenericTree(x, 0, showids, labels, showtype, TYPE_MARGIN) + + else: + printGenericTree(element, 0, showids, labels, showtype, TYPE_MARGIN) diff --git a/ontospy/core/rdf_loader.py b/ontospy/core/rdf_loader.py index 1eb25544..db0bbfe6 100755 --- a/ontospy/core/rdf_loader.py +++ b/ontospy/core/rdf_loader.py @@ -20,13 +20,12 @@ class RDFLoader(object): - """ - Utility to Load any RDF source into an RDFLIB graph instance. + """Utility to Load any RDF source into an RDFLIB graph instance. Accepts: [single item or list] :: uri_or_path = a uri or local path :: data = a string containing rdf - :: file_obj = a python file objecy + :: file_obj = a python file object Returns: rdflib graph instance. @@ -112,12 +111,11 @@ def load(self, uri_or_path=None, data=None, file_obj=None, rdf_format=""): return self.rdflib_graph def load_uri(self, uri): - """ - Load a single resource into the graph for this object. + """Load a single resource into the graph for this object. Approach: try loading into a temporary graph first, if that succeeds merge it into the main graph. This allows to deal with the JSONLD loading issues which can solved only by using a ConjunctiveGraph (https://github.com/RDFLib/rdflib/issues/436). Also it deals with the RDFA error message which seems to stick into a graph even if the parse operation fails. - NOTE the final merge operation can be improved as graph-set operations involving blank nodes could case collisions (https://rdflib.readthedocs.io/en/stable/merging.html) + TODO the final merge operation can be improved as graph-set operations involving blank nodes could case collisions (https://rdflib.readthedocs.io/en/stable/merging.html) :param uri: single RDF source location :return: None (sets self.rdflib_graph and self.sources_valid) @@ -258,7 +256,7 @@ def loading_failed(self, rdf_format_opts, uri=""): "----------\nFatal error parsing graph%s\n(using RDF serializations: %s)" % (uri, str(rdf_format_opts)), "red") printDebug( - "----------\nTIP: You can try one of the following RDF validation services\n\n" + "----------\nTIP: You can try one of the following RDF validation services\n\n\n" ) return diff --git a/ontospy/core/utils.py b/ontospy/core/utils.py index 2dd41202..a3490ed3 100755 --- a/ontospy/core/utils.py +++ b/ontospy/core/utils.py @@ -877,13 +877,13 @@ def slugify(value): def get_files_with_extensions(folder, extensions): - """walk dir and return .* files as a list + """walk dir and return .* files as a list. Hidden files are ignored. Note: directories are walked recursively""" out = [] for root, dirs, files in os.walk(folder): for file in files: filename, file_extension = os.path.splitext(file) - if file_extension.replace(".", "") in extensions: + if not filename.startswith('.') and file_extension.replace(".", "") in extensions: out += [os.path.join(root, file)] # break diff --git a/ontospy/ontodocs/CONFIG.py b/ontospy/ontodocs/CONFIG.py index 6ad20af6..3c426522 100755 --- a/ontospy/ontodocs/CONFIG.py +++ b/ontospy/ontodocs/CONFIG.py @@ -18,7 +18,7 @@ "Description": "@todo", }, { - "ID": "d3-tree", + "ID": "d3-dendogram", "Title": "D3 Dendogram", "Description": "@todo", }, diff --git a/ontospy/ontodocs/builder.py b/ontospy/ontodocs/builder.py index ac6a0fe0..84744f6d 100755 --- a/ontospy/ontodocs/builder.py +++ b/ontospy/ontodocs/builder.py @@ -177,28 +177,28 @@ def build_visualization(ontouri, g, viz_index, path=None, title="", theme=""): from .viz.viz_markdown import MarkdownViz v = MarkdownViz(g, title) - elif this_viz['ID'] == "d3-tree": - from .viz.viz_d3tree import Dataviz + elif this_viz['ID'] == "d3-dendogram": + from .viz.viz_d3dendogram import Dataviz v = Dataviz(g, title) elif this_viz['ID'] == "d3-bubble-chart": - from .viz.viz_d3bubbleChart import Dataviz + from .viz.viz_d3bubble_chart import Dataviz v = Dataviz(g, title) elif this_viz['ID'] == "d3-pack-hierarchy": - from .viz.viz_d3packHierarchy import Dataviz + from .viz.viz_d3pack_hierarchy import Dataviz v = Dataviz(g, title) elif this_viz['ID'] == "d3-bar-hierarchy": - from .viz.viz_d3barHierarchy import Dataviz + from .viz.viz_d3bar_hierarchy import Dataviz v = Dataviz(g, title) elif this_viz['ID'] == "d3-partition-table": - from .viz.viz_d3partitionTable import Dataviz + from .viz.viz_d3partition_table import Dataviz v = Dataviz(g, title) elif this_viz['ID'] == "d3-rotating-cluster": - from .viz.viz_d3rotatingCluster import Dataviz + from .viz.viz_d3rotating_cluster import Dataviz v = Dataviz(g, title) elif this_viz['ID'] == "sigma-force-directed": diff --git a/ontospy/ontodocs/media/templates/d3/d3_barHierarchy.html b/ontospy/ontodocs/media/templates/d3/d3_bar_hierarchy.html similarity index 91% rename from ontospy/ontodocs/media/templates/d3/d3_barHierarchy.html rename to ontospy/ontodocs/media/templates/d3/d3_bar_hierarchy.html index afb5aaaa..796f2f8d 100755 --- a/ontospy/ontodocs/media/templates/d3/d3_barHierarchy.html +++ b/ontospy/ontodocs/media/templates/d3/d3_bar_hierarchy.html @@ -1,12 +1,9 @@ - - - +{% extends "d3base.html" %} -{{ontology.uri}} - +{% block custom_css %} +{% endblock custom_css %} - - - {# VISIBLE PAGE HTML #} - {# note: showpanel arg is the ID of the div that is showing.. #} +{% block main_content %} -

Ontology: {{main_uri}}

-

{{ontology.bestDescription}}

-

Ontology documentation automatically generated by Ontospy on {% now "jS F Y H:i" %}

- -
{% if TOTAL_CLASSES %} @@ -93,13 +83,20 @@

Classes ({{TOTAL_CLASSES}})

-
- Documentation automatically generated by Ontospy on {% now "jS F Y H:i" %} +{% endblock main_content %} + + + + + +{% block the_javascript %} + + {# JAVASCRIPT #} @@ -109,6 +106,17 @@

Classes ({{TOTAL_CLASSES}})

+ + + + + +{% include 'js_slidereveal.html' %} + + + + + + + + + +{% include 'js_slidereveal.html' %} + + + + + #} - - {# JAVASCRIPT #} @@ -353,7 +347,6 @@

Concepts ({{TOTAL_CONCEPTS}})

.append("svg:g") .attr("transform", "translate(" + m[3] + "," + m[0] + ")"); - // d3.json(JSON_FILE, function(json) { root_classes = json1; root_classes.x0 = h / 2; root_classes.y0 = 0; @@ -363,6 +356,7 @@

Concepts ({{TOTAL_CONCEPTS}})

// Initialize the display to show no nodes: root_classes.children.forEach(toggleAll); + {% endif %} @@ -408,7 +402,6 @@

Concepts ({{TOTAL_CONCEPTS}})

.append("svg:g") .attr("transform", "translate(" + m[3] + "," + m[0] + ")"); - // d3.json(JSON_FILE, function(json) { root_properties = json2; root_properties.x0 = h / 2; root_properties.y0 = 0; diff --git a/ontospy/ontodocs/media/templates/d3/d3_dendogram.html b/ontospy/ontodocs/media/templates/d3/d3_dendogram.html new file mode 100755 index 00000000..19f0c109 --- /dev/null +++ b/ontospy/ontodocs/media/templates/d3/d3_dendogram.html @@ -0,0 +1,554 @@ +{% extends "d3base.html" %} + + + + +{% block custom_css %} + + + + + +{% endblock custom_css %} + + + + + + + +{% block main_content %} + + + + {% if TOTAL_CLASSES %} +
+

Classes ({{TOTAL_CLASSES}})

+
+ + +
+ + +
+ {% endif %} + + {% if TOTAL_PROPERTIES %} +
+

Properties ({{TOTAL_PROPERTIES}})

+
+ + +
+ + +
+ {% endif %} + + + {% if TOTAL_CONCEPTS %} + +
+

Concepts ({{TOTAL_CONCEPTS}})

+
+ + +
+ + +
+ + {% endif %} + + + + +{% endblock main_content %} + + + + + + + + + + +{% block the_javascript %} + + + + + + {# JAVASCRIPT #} + + {% if save_on_github %} + + + + + + + + {% else %} + + + + + + + + + {% endif %} + + + + + {% include 'js_slidereveal.html' %} + + + + + +{# JAVASCRIPT FOR D3 TREE #} + + + + + + + + +{% endblock the_javascript %} + + + + diff --git a/ontospy/ontodocs/media/templates/d3/d3_pack_hierarchy.html b/ontospy/ontodocs/media/templates/d3/d3_pack_hierarchy.html new file mode 100755 index 00000000..9c02e959 --- /dev/null +++ b/ontospy/ontodocs/media/templates/d3/d3_pack_hierarchy.html @@ -0,0 +1,261 @@ +{% extends "d3base.html" %} + + + + +{% block custom_css %} + + + + + +{% endblock custom_css %} + + + + + + + + +{% block main_content %} + + + + + {% if TOTAL_CLASSES %} +
+

Classes ({{TOTAL_CLASSES}})

+
+ + +
+ + +
+ {% endif %} + + + +{% endblock main_content %} + + + + + +{% block the_javascript %} + + + + + + + + + + + + + {% include 'js_slidereveal.html' %} + + + + + + + + +{% endblock the_javascript %} + + diff --git a/ontospy/ontodocs/media/templates/d3/d3_partitionTable.html b/ontospy/ontodocs/media/templates/d3/d3_partition_table.html similarity index 85% rename from ontospy/ontodocs/media/templates/d3/d3_partitionTable.html rename to ontospy/ontodocs/media/templates/d3/d3_partition_table.html index db74f139..0f76b449 100755 --- a/ontospy/ontodocs/media/templates/d3/d3_partitionTable.html +++ b/ontospy/ontodocs/media/templates/d3/d3_partition_table.html @@ -1,12 +1,10 @@ - - - +{% extends "d3base.html" %} -{{ontology.uri}} - +{% block custom_css %} + - +{% endblock custom_css %} + + - - {# VISIBLE PAGE HTML #} - {# note: showpanel arg is the ID of the div that is showing.. #} -

Ontology: {{main_uri}}

-

{{ontology.bestDescription}}

-

Ontology documentation automatically generated by Ontospy on {% now "jS F Y H:i" %}

-
+{% block main_content %} + {% if TOTAL_CLASSES %} @@ -159,8 +153,10 @@

Classes ({{TOTAL_CLASSES}})

-
- Documentation automatically generated by Ontospy on {% now "jS F Y H:i" %} + + {% endblock main_content %} + + @@ -168,6 +164,12 @@

Classes ({{TOTAL_CLASSES}})

+{% block the_javascript %} + + + + + {# JAVASCRIPT #} @@ -175,6 +177,17 @@

Classes ({{TOTAL_CLASSES}})

+ + + + + +{% include 'js_slidereveal.html' %} + + + + + + + + + +{% include 'js_slidereveal.html' %} + + + + + + + + +{% include 'js_slidereveal.html' %} + + + + + + + + diff --git a/ontospy/ontodocs/media/templates/html-multi/breadcrumbs.html b/ontospy/ontodocs/media/templates/html-multi/breadcrumbs.html index bfaa2cb8..43a4f0f1 100755 --- a/ontospy/ontodocs/media/templates/html-multi/breadcrumbs.html +++ b/ontospy/ontodocs/media/templates/html-multi/breadcrumbs.html @@ -124,7 +124,7 @@ Classes
  • - {{each.uri}} + {{each.title}}
    • @@ -147,7 +147,7 @@ Properties
  • - {{each.uri}} + {{each.title}}
    • @@ -170,7 +170,7 @@ Concepts
  • - {{each.uri}} + {{each.title}}
    • @@ -192,7 +192,7 @@ Shapes
  • - {{each.uri}} + {{each.title}}
    • diff --git a/ontospy/ontodocs/media/templates/html-multi/browser/browser_classinfo.html b/ontospy/ontodocs/media/templates/html-multi/browser/browser_classinfo.html index dcbb6f25..1371f3bd 100755 --- a/ontospy/ontodocs/media/templates/html-multi/browser/browser_classinfo.html +++ b/ontospy/ontodocs/media/templates/html-multi/browser/browser_classinfo.html @@ -42,7 +42,7 @@

    {# Class: #} - {{each.qname}} + {{each.title}} {% if not each.children %} leaf node {% endif %} @@ -61,14 +61,14 @@

  • - {{s.qname}} + {{s.title}}
      -
    • {{each.qname}} +
    • {{each.title}} {% if each.children %} {% endif %}
      • @@ -83,11 +83,11 @@

    • owl:Thing
        -
      • {{each.qname}} +
      • {{each.title}} {% if each.children %} {% endif %}
        • @@ -140,7 +140,7 @@

        Description

        - {{each.bestDescription|default:"--"}} + {{each.bestDescription|linebreaks|default:"--"}}
        @@ -158,7 +158,7 @@

        Superclasses ({{each.ancestors|length}})

        {% if each.ancestors %} - {% for s in each.ancestors %}
      • {{s.qname}}
        • {% endfor %} + {% for s in each.ancestors %}
      • {{s.title}}
        • {% endfor %} {% else %}
      • owl:Thing
        • {% endif %} @@ -179,7 +179,7 @@

        Shapes ({{each.all_shapes|length}})

        - {% for s in each.all_shapes%}
      • {{s.qname}}
        • {% endfor %} + {% for s in each.all_shapes%}
      • {{s.title}}
        • {% endfor %}
        @@ -197,7 +197,7 @@

        Usage

        -

        Instances of {{each.qname}} can have the following properties:

        +

        Instances of {{each.title}} can have the following properties:

        @@ -212,14 +212,14 @@

        Usage

        {% if v %} - {% for prop in v %}
        From class {{k.qname}} + From class {{k.title}}
        - {{prop.qname}} + {{prop.title}} {{prop.rdftype_qname}} @@ -232,9 +232,9 @@

        Usage

        {% for range in prop.ranges %} {% if not range.ext_model %} - {{range.qname}} + {{range.title}} {% else %} - {{range.qname}} + {{range.title}} {% endif %} diff --git a/ontospy/ontodocs/media/templates/html-multi/browser/browser_conceptinfo.html b/ontospy/ontodocs/media/templates/html-multi/browser/browser_conceptinfo.html index 1c20834e..3ed34e13 100755 --- a/ontospy/ontodocs/media/templates/html-multi/browser/browser_conceptinfo.html +++ b/ontospy/ontodocs/media/templates/html-multi/browser/browser_conceptinfo.html @@ -40,7 +40,7 @@

        {# Concept: #} - {{each.qname}} + {{each.title}} {% if not each.children %} leaf node {% endif %} @@ -61,14 +61,14 @@

      • - {{s.qname}} + {{s.title}}
          -
        • {{each.qname}} +
        • {{each.title}} {% if each.children %} {% endif %}
          • @@ -83,11 +83,11 @@

        • skos:Concept
            -
          • {{each.qname}} +
          • {{each.title}} {% if each.children %} {% endif %}
            • @@ -143,7 +143,7 @@

            Description

            - {{each.bestDescription|default:"--"}} + {{each.bestDescription|linebreaks|default:"--"}}
            @@ -161,7 +161,7 @@

            Inherits from

            {% if each.ancestors %} - {% for s in each.ancestors %}
          • {{s.qname}}
            • {% endfor %} + {% for s in each.ancestors %}
          • {{s.title}}
            • {% endfor %} {% else %}
          • owl:Thing
            • {% endif %} diff --git a/ontospy/ontodocs/media/templates/html-multi/browser/browser_entities_az.html b/ontospy/ontodocs/media/templates/html-multi/browser/browser_entities_az.html index eb090df3..06fff1a0 100755 --- a/ontospy/ontodocs/media/templates/html-multi/browser/browser_entities_az.html +++ b/ontospy/ontodocs/media/templates/html-multi/browser/browser_entities_az.html @@ -57,7 +57,7 @@

            Classes ({{ ontospy_graph.all_classes|length }})

            {% for each in ontospy_graph.all_classes %} -
            {{each.qname}}
            +
            {{each.title}}
            {# {{ each.bestDescription|default:"[no definition]" }}#}
            @@ -84,7 +84,7 @@

            Concepts ({{ ontospy_graph.all_skos_concepts|length }})<
            {% for each in ontospy_graph.all_skos_concepts %} -
            {{each.qname}}
            +
            {{each.title}}
            {# {{ each.bestDescription|default:"[no definition]" }}#}
            @@ -120,7 +120,7 @@

            Properties ({{ ontospy_graph.all_properties|length }}) {% for each in ontospy_graph.all_properties %} -
            {{each.qname}}
            +
            {{each.title}}
            {# {{ each.bestDescription|default:"[no definition]" }}#}
            @@ -149,7 +149,7 @@

            Shapes ({{ ontospy_graph.all_shapes|length }})

            {% for each in ontospy_graph.all_shapes %} -
            {{each.qname}}
            +
            {{each.title}}
            {# {{ each.bestDescription|default:"[no definition]" }}#}
            diff --git a/ontospy/ontodocs/media/templates/html-multi/browser/browser_propinfo.html b/ontospy/ontodocs/media/templates/html-multi/browser/browser_propinfo.html index 55db5c6d..c0e3ef16 100755 --- a/ontospy/ontodocs/media/templates/html-multi/browser/browser_propinfo.html +++ b/ontospy/ontodocs/media/templates/html-multi/browser/browser_propinfo.html @@ -36,7 +36,7 @@

            {# Property: #} - {{each.qname}} + {{each.title}} {% if not each.children %} leaf node {% endif %} @@ -56,14 +56,14 @@

          • - {{s.qname}} + {{s.title}}

            • - {{each.bestDescription|default:"--"}} + {{each.bestDescription|linebreaks|default:"--"}}
            @@ -154,7 +154,7 @@

            Inherits from

            {% if each.ancestors %} - {% for s in each.ancestors %}
          • {{s.qname}}
            • {% endfor %} + {% for s in each.ancestors %}
          • {{s.title}}
            • {% endfor %} {% else %}
          • owl:Thing
            • {% endif %} @@ -176,7 +176,7 @@

            Sub Property

            - {% for s in each.children %}
          • {{s.qname}}
            • {% endfor %} + {% for s in each.children %}
          • {{s.title}}
            • {% endfor %}
            @@ -202,11 +202,11 @@

            Usage

            {% if each.domains %} {% for s in each.domains %} {% if s.ext_model %} - {{s.qname|default:s}} + {{s.title|default:s}} {% elif s.is_Bnode %} Blank node (see implementation) {% else %} - {{s.qname|default:s}} + {{s.title|default:s}} {% endif %} {% if not forloop.last %}, {% endif %} {% endfor %} @@ -216,18 +216,18 @@

            Usage

        		•  - {{each.qname}} + {{each.title}} {% if each.ranges %} {% for s in each.ranges %} {% if s.ext_model %} - {{s.qname|default:s}} + {{s.title|default:s}} {% elif s.is_Bnode %} Blank node (see implementation) {% else %} - {{s.qname|default:s}} + {{s.title|default:s}} {% endif %} {% if not forloop.last %}, {% endif %} {% endfor %} diff --git a/ontospy/ontodocs/media/templates/html-multi/browser/browser_shapeinfo.html b/ontospy/ontodocs/media/templates/html-multi/browser/browser_shapeinfo.html index 44d161a8..e800dc76 100644 --- a/ontospy/ontodocs/media/templates/html-multi/browser/browser_shapeinfo.html +++ b/ontospy/ontodocs/media/templates/html-multi/browser/browser_shapeinfo.html @@ -42,7 +42,7 @@

        {# Class: #} - {{each.qname}} + {{each.title}} {% if not each.children %} leaf node {% endif %} @@ -61,14 +61,14 @@

      • - {{s.qname}} + {{s.title}}
          -
        • {{each.qname}} +
        • {{each.title}} {% if each.children %} {% endif %}
          • @@ -83,11 +83,11 @@

        • sh:Shape
            -
          • {{each.qname}} +
          • {{each.title}} {% if each.children %} {% endif %}
            • @@ -140,7 +140,7 @@

            Description

            - {{each.bestDescription|default:"--"}} + {{each.bestDescription|linebreaks|default:"--"}}
            @@ -158,7 +158,7 @@

            Superclasses ({{each.ancestors|length}})

            {% if each.ancestors %} - {% for s in each.ancestors %}
          • {{s.qname}}
            • {% endfor %} + {% for s in each.ancestors %}
          • {{s.title}}
            • {% endfor %} {% else %}
          • owl:Thing
            • {% endif %} @@ -177,7 +177,7 @@

            Target Classes ({{each.targetClasses|length}})

            - {% for s in each.targetClasses %}
          • {{s.qname}}
            • {% endfor %} + {% for s in each.targetClasses %}
          • {{s.title}}
            • {% endfor %}
            diff --git a/ontospy/ontodocs/media/templates/html-multi/navbar.html b/ontospy/ontodocs/media/templates/html-multi/navbar.html index 350ea86a..8fab3fe4 100755 --- a/ontospy/ontodocs/media/templates/html-multi/navbar.html +++ b/ontospy/ontodocs/media/templates/html-multi/navbar.html @@ -18,10 +18,6 @@