From 2c1bae150e6b86f7fda3781c0e4d8a2bcfdf79da Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 17 Apr 2024 03:07:04 +0200 Subject: [PATCH] ontdoc update and refactoring --- dialogs/tool/convertlayerdialog.py | 8 +- tabs/interlinkingtab.py | 10 +- tasks/processing/convertlayertask.py | 2 +- tasks/processing/ontdoctask.py | 6 +- tasks/query/data/querysubgraphtask.py | 2 +- tasks/query/discovery/classtreequerytask.py | 67 +- tasks/query/enrichment/enrichmentquerytask.py | 2 +- util/doc/classtreeutils.py | 169 ++ util/doc/docconfig.py | 4 +- util/doc/docdefaults.py | 6 +- util/doc/docutils.py | 133 +- util/doc/graphutils.py | 259 +++ util/doc/literalutils.py | 28 + util/doc/ontdocgeneration.py | 1584 ++--------------- util/doc/templateutils.py | 16 + util/export/api/ckanexporter.py | 128 +- util/export/api/iiifexporter.py | 80 +- util/export/api/solidexporter.py | 2 +- util/export/data/exporter/layer/__init__.py | 0 util/export/data/exporter/rdf/__init__.py | 0 util/export/data/exporter/rdf/voidexporter.py | 140 -- .../data/{exporter/rdf => }/geoexporter.py | 2 +- .../data/{exporter/rdf => }/graphexporter.py | 2 +- util/export/data/htmlexporter.py | 900 ++++++++++ .../data/{exporter/rdf => }/miscexporter.py | 2 +- util/export/data/voidexporter.py | 175 ++ .../data/{exporter/rdf => }/vowlexporter.py | 5 +- .../{data/exporter => }/exporterutils.py | 6 +- .../{data/exporter => layer}/__init__.py | 0 .../exporter => }/layer/layerexporter.py | 11 +- util/export/pages/indexviewpage.py | 165 ++ util/export/pages/owltimepage.py | 4 +- util/export/pages/page.py | 22 + util/export/srs/crsexporttools.py | 78 +- util/graphutils.py | 8 +- util/sparqlutils.py | 4 +- util/ui/uiutils.py | 2 +- 37 files changed, 2303 insertions(+), 1729 deletions(-) create mode 100644 util/doc/classtreeutils.py create mode 100644 util/doc/graphutils.py create mode 100644 util/doc/literalutils.py delete mode 100644 util/export/data/exporter/layer/__init__.py delete mode 100644 util/export/data/exporter/rdf/__init__.py delete mode 100644 util/export/data/exporter/rdf/voidexporter.py rename util/export/data/{exporter/rdf => }/geoexporter.py (99%) rename util/export/data/{exporter/rdf => }/graphexporter.py (99%) create mode 100644 util/export/data/htmlexporter.py rename util/export/data/{exporter/rdf => }/miscexporter.py (98%) create mode 100644 util/export/data/voidexporter.py rename util/export/data/{exporter/rdf => }/vowlexporter.py (98%) rename util/export/{data/exporter => }/exporterutils.py (91%) rename util/export/{data/exporter => layer}/__init__.py (100%) rename util/export/{data/exporter => }/layer/layerexporter.py (98%) create mode 100644 util/export/pages/indexviewpage.py create mode 100644 util/export/pages/page.py diff --git a/dialogs/tool/convertlayerdialog.py b/dialogs/tool/convertlayerdialog.py index ad70caf5c..fb15e8158 100644 --- a/dialogs/tool/convertlayerdialog.py +++ b/dialogs/tool/convertlayerdialog.py @@ -1,12 +1,12 @@ from qgis.PyQt import uic from qgis.PyQt import QtWidgets -from qgis.core import QgsProject,QgsMapLayerProxyModel, Qgis -from qgis.PyQt.QtWidgets import QProgressDialog, QFileDialog,QMessageBox +from qgis.core import QgsMapLayerProxyModel +from qgis.PyQt.QtWidgets import QProgressDialog, QFileDialog from qgis.core import QgsApplication -from qgis.core import Qgis,QgsTask, QgsMessageLog +from qgis.core import Qgis, QgsMessageLog from qgis.PyQt.QtCore import Qt -from ...util.export.data.exporter.exporterutils import ExporterUtils +from ...util.export.exporterutils import ExporterUtils from ...util.ui.uiutils import UIUtils from ...tasks.processing.convertlayertask import ConvertLayerTask import os.path diff --git a/tabs/interlinkingtab.py b/tabs/interlinkingtab.py index b24ea2300..b1b780cb3 100644 --- a/tabs/interlinkingtab.py +++ b/tabs/interlinkingtab.py @@ -12,7 +12,7 @@ from ..util.interlinkutils import InterlinkUtils from ..util.ui.uiutils import UIUtils -from ..util.export.data.exporter.exporterutils import ExporterUtils +from ..util.export.exporterutils import ExporterUtils from ..tasks.processing.convertlayertask import ConvertLayerTask ## Provides implementations for functions accessible from the interlinking tab @@ -83,8 +83,8 @@ def suggestSchema(self): for col in columntypes: if "id" in col and col["id"] and not col["geotype"]: self.dlg.interlinkTable.item(counter,1).setCheckState(col["id"]) - item = QTableWidgetItem("rdf:type") - item.setText("rdf:type") + item = QTableWidgetItem("data:type") + item.setText("data:type") item.setData(1, "http://www.w3.org/1999/02/22-rdf-syntax-ns#type") self.dlg.interlinkTable.setItem(counter, 4,item) if col["geotype"]: @@ -199,8 +199,8 @@ def readMapping(self, filename): if columnname not in filedata["columns"]: if "indid" in filedata and filedata["indid"]==columnname: self.dlg.interlinkTable.item(row, 1).setCheckState(True) - item = QTableWidgetItem("rdf:type") - item.setText("rdf:type") + item = QTableWidgetItem("data:type") + item.setText("data:type") item.setData(1, "http://www.w3.org/1999/02/22-rdf-syntax-ns#type") self.dlg.interlinkTable.setItem(row, 4, item) comboboxx = self.dlg.interlinkTable.cellWidget(row, 5) diff --git a/tasks/processing/convertlayertask.py b/tasks/processing/convertlayertask.py index 962ba858c..899ef84f0 100644 --- a/tasks/processing/convertlayertask.py +++ b/tasks/processing/convertlayertask.py @@ -1,5 +1,5 @@ -from ...util.export.data.exporter.layer.layerexporter import LayerExporter +from ...util.export.layer.layerexporter import LayerExporter from qgis.utils import iface from qgis.core import Qgis,QgsTask, QgsMessageLog from qgis.PyQt.QtWidgets import QMessageBox diff --git a/tasks/processing/ontdoctask.py b/tasks/processing/ontdoctask.py index 28cac1bcc..78bdc1439 100644 --- a/tasks/processing/ontdoctask.py +++ b/tasks/processing/ontdoctask.py @@ -21,10 +21,8 @@ def __init__(self, description, graphname, namespace,prefixes,license,labellang, self.startconcept=startconcept self.offlinecompat=offlinecompat self.exports=exports - self.ogcapifeatures=ogcapifeatures - self.iiif=iiif self.imagemetadata=imagemetadata - self.ckan=ckan + self.apis={"ogcapifeatures":ogcapifeatures,"iiif":iiif,"ckan":ckan,"solidexport":False} self.createVOWL=createVOWL self.createMetadataTable=createMetadataTable self.nonNSPagesCBox=nonNSPagesCBox @@ -55,7 +53,7 @@ def run(self): else: nsshort=self.namespace[self.namespace.rfind('/') + 1:] #try: - ontdoc=OntDocGeneration(self.prefixes, self.namespace, nsshort,self.license,self.labellang, self.outpath, self.graph,self.createcollections,self.baselayers,self.tobeaddedperInd,self.maincolor,self.titlecolor,self.progress,self.createIndexPages,self.nonNSPagesCBox,self.createMetadataTable,self.createVOWL,self.ogcapifeatures,self.iiif,self.ckan,self.imagemetadata,self.startconcept,self.deploymenturl,self.logopath,self.offlinecompat,self.exports) + ontdoc=OntDocGeneration(self.prefixes, self.namespace, nsshort,self.license,self.labellang, self.outpath, self.graph,self.createcollections,self.baselayers,self.tobeaddedperInd,self.maincolor,self.titlecolor,self.progress,self.createIndexPages,self.nonNSPagesCBox,self.createMetadataTable,self.createVOWL,self.apis,self.imagemetadata,self.startconcept,self.deploymenturl,self.logopath,self.offlinecompat,self.exports) ontdoc.generateOntDocForNameSpace(self.namespace) #except Exception as e: # self.exception=e diff --git a/tasks/query/data/querysubgraphtask.py b/tasks/query/data/querysubgraphtask.py index b25ca0a3f..59830dea9 100644 --- a/tasks/query/data/querysubgraphtask.py +++ b/tasks/query/data/querysubgraphtask.py @@ -1,6 +1,6 @@ from rdflib import Graph -from ....util.export.data.exporter.exporterutils import ExporterUtils +from ....util.export.exporterutils import ExporterUtils from ....util.sparqlutils import SPARQLUtils from qgis.core import Qgis,QgsTask, QgsMessageLog from qgis.PyQt.QtWidgets import QFileDialog diff --git a/tasks/query/discovery/classtreequerytask.py b/tasks/query/discovery/classtreequerytask.py index a38aed866..ac7a9b8e6 100644 --- a/tasks/query/discovery/classtreequerytask.py +++ b/tasks/query/discovery/classtreequerytask.py @@ -37,75 +37,12 @@ def __init__(self, description, triplestoreurl,dlg,treeNode,triplestoreconf,pref self.classtreemap=None self.subclassmap=None self.query="""PREFIX owl: \n - PREFIX rdf: \n + PREFIX data: \n PREFIX rdfs: \n SELECT DISTINCT ?subject ?label ?supertype \n WHERE {\n""" - self.query += "{ ?individual <" + str(self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]["typeproperty"]) + "> ?subject . } UNION { ?subject <" + str(self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]["typeproperty"]) + "> owl:Class . } UNION { ?subject <" + str(self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]["typeproperty"]) + "> rdfs:Class . } \n" + self.query += "{ ?individual <" + str(self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]["typeproperty"]) + "> ?subject . } UNION { ?subject <" + str(self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]["typeproperty"]) + "> owl:Class . } UNION { ?subject <" + str(self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]["typeproperty"]) + "> rdfs:Class . } UNION { ?subject <" + str(self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]["subclassproperty"]) + "> ?supertype . } \n" self.query += """OPTIONAL { ?subject <""" + str(self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]["subclassproperty"]) + """> ?supertype . }\n""" + SPARQLUtils.resolvePropertyToTriplePattern("%%labelproperty%%", "?label", "?subject",self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()],"OPTIONAL", "")+ """ }""" - """ - self.geoquery=PREFIX owl: \n - PREFIX rdf: \n - PREFIX rdfs: \n - SELECT DISTINCT ?subject ?label ?supertype (Bound(?hasgeo) AS ?hgeo)\n - - - self.optionalpart="" - if "geotriplepattern" in self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]: - geotriplepattern="" - if len(self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]["geotriplepattern"])>1: - geotriplepattern+="OPTIONAL {" - first=True - for geopat in self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]["geotriplepattern"]: - if first: - first=False - else: - geotriplepattern+=" UNION " - geotriplepattern += " { " + geopat.replace("?geo", "?hasgeo").replace("?lat", "?hasgeo").replace("?item","?individual") + " }\n" - geotriplepattern+="}\n" - elif len(self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]["geotriplepattern"])>0: - geotriplepattern += "OPTIONAL { " + self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]["geotriplepattern"][0].replace("?geo", "?hasgeo").replace("?item","?individual")+" }\n" - self.optionalpart=geotriplepattern - if "highload" in self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()] and self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]["highload"]: - self.query+="{ ?individual <"+self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]["typeproperty"]+"> ?subject . } UNION { ?subject <"+str(self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]["typeproperty"])+"> owl:Class . } \n" - elif self.optionalpart=="" and "geometryproperty" in self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()] \ - and (self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]["geometryproperty"]=="http://www.opengis.net/ont/geosparql#asWKT" - or self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]["geometryproperty"]=="http://www.opengis.net/ont/geosparql#asGML" - or self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]["geometryproperty"]=="http://www.opengis.net/ont/geosparql#asGeoJSON" - or self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]["geometryproperty"]=="http://www.opengis.net/ont/geosparql#asKML" - ): - if isinstance(self.triplestoreconf["geometryproperty"], str): - self.optionalpart = "OPTIONAL {BIND(EXISTS {?individual ?a ?lit . ?lit <" + str( - self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()][ - "geometryproperty"]) + "> ?wkt } AS ?hasgeo)}" - else: - self.optionalpart = "OPTIONAL {?individual ?a ?lit . ?lit <" + str( - self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()][ - "geometryproperty"][0]) + "> ?hasgeo }" - #self.query+="{ ?individual <"+str(self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]["typeproperty"])+"> ?subject . "+str(self.optionalpart)+"} UNION { ?subject <"+str(self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]["typeproperty"])+"> owl:Class . } \n" - elif self.optionalpart=="" and "geometryproperty" in self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()] and self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]["geometryproperty"]=="http://www.opengis.net/ont/geosparql#hasGeometry": - if isinstance(self.triplestoreconf["geometryproperty"], str): - self.optionalpart = "OPTIONAL {BIND(EXISTS {?individual <" + str( - self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()][ - "geometryproperty"]) + "> ?lit . ?lit ?a ?wkt } AS ?hasgeo)}" - else: - self.optionalpart = "OPTIONAL {?individual <" + str( - self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()][ - "geometryproperty"][0]) + "> ?lit . ?lit ?a ?hasgeo }" - elif self.optionalpart=="" and "geometryproperty" in self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()] and self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]["geometryproperty"]!="": - if isinstance(self.triplestoreconf["geometryproperty"],str): - self.optionalpart = "OPTIONAL {BIND(EXISTS {?individual <" + str( - self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()][ - "geometryproperty"]) + "> ?wkt } AS ?hasgeo)}" - elif isinstance(self.triplestoreconf["geometryproperty"],list): - self.optionalpart = "OPTIONAL {?individual <" + str( - self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()][ - "geometryproperty"][0]) + "> ?hasgeo }" - #self.query+="{ ?individual <"+self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]["typeproperty"]+"> ?subject . "+str(self.optionalpart)+"} UNION { ?subject <"+str(self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]["typeproperty"])+"> owl:Class . } \n" - elif self.optionalpart=="": - #self.query += "{ ?individual <" + self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]["typeproperty"] + "> ?subject . } UNION { ?subject <" + str(self.dlg.triplestoreconf[self.dlg.comboBox.currentIndex()]["typeproperty"]) + "> owl:Class . } .\n" - self.query=self.query.replace("(Bound(?hasgeo) AS ?hgeo)","").replace("?hasgeo","") - """ def run(self): diff --git a/tasks/query/enrichment/enrichmentquerytask.py b/tasks/query/enrichment/enrichmentquerytask.py index 447fd3d99..d0bdd71d3 100644 --- a/tasks/query/enrichment/enrichmentquerytask.py +++ b/tasks/query/enrichment/enrichmentquerytask.py @@ -102,7 +102,7 @@ def run(self): if self.table.item(self.row, 7).text() != "" and "wikidata" in curtriplestoreconf["resource"]["url"]: query += "?item wdt:P31 <" + str(self.table.item(self.row, 7).text()) + "> .\n" else: - query += "?item rdf:type <" + str(self.table.item(self.row, 7).text()) + "> .\n" + query += "?item data:type <" + str(self.table.item(self.row, 7).text()) + "> .\n" query += "?item <" + str(self.idprop) + "> ?vals .\n" query += "?item <" + str(proppp) + "> ?val . \n" if (self.content == "Enrich Value" or self.content == "Enrich Both") and not "wikidata" in curtriplestoreconf["resource"]["url"]: diff --git a/util/doc/classtreeutils.py b/util/doc/classtreeutils.py new file mode 100644 index 000000000..e8fc1d4e8 --- /dev/null +++ b/util/doc/classtreeutils.py @@ -0,0 +1,169 @@ +from .docconfig import DocConfig +from .docutils import DocUtils +from ..sparqlutils import SPARQLUtils +from rdflib import URIRef + +class ClassTreeUtils: + + + @staticmethod + def getClassTree(graph, uritolabel, classidset, uritotreeitem,typeproperty,prefixes,preparedclassquery): + results = graph.query(preparedclassquery) + ldcontext = {"@context": { + "@version": 1.1, + "foaf": "http://xmlns.com/foaf/0.1/", + "ct": "http://purl.org/vocab/classtree#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "icon": "foaf:image", + "id": "@id", + "parent": "rdfs:subClassOf", + "halfgeoclass": "ct:HalfGeoClass", + "geoclass": {"@type": "ct:icontype", "@id": "ct:GeoClass"}, + "collectionclass": {"@type": "ct:icontype", "@id": "ct:CollectionClass"}, + "featurecollectionclass": {"@type": "ct:icontype", "@id": "ct:FeatureCollectionClass"}, + "class": "owl:Class", + "instance": "owl:NamedIndividual", + "geoinstance": {"@type": "ct:Icontype", "@id": "ct:GeoNamedIndividual"}, + "text": "rdfs:label", + "type": "ct:icontype", + "types": "ct:icontypes", + "core": {"@type": "ct:TreeConfig", "@id": "@nest"}, + "data": {"@id": "ct:treeitem", "@type": "ct:TreeItem"} + }} + tree = {"plugins": ["defaults", "search", "sort", "state", "types", "contextmenu"], + "search": {"show_only_matches": True}, "types": { + "class": {"icon": "https://cdn.jsdelivr.net/gh/i3mainz/geopubby@master/public/icons/class.png"}, + "geoclass": {"icon": "https://cdn.jsdelivr.net/gh/i3mainz/geopubby@master/public/icons/geoclass.png"}, + "halfgeoclass": { + "icon": "https://cdn.jsdelivr.net/gh/i3mainz/geopubby@master/public/icons/halfgeoclass.png"}, + "collectionclass": { + "icon": "https://cdn.jsdelivr.net/gh/i3mainz/geopubby@master/public/icons/collectionclass.png"}, + "geocollection": { + "icon": "https://cdn.jsdelivr.net/gh/i3mainz/geopubby@master/public/icons/geometrycollection.png"}, + "featurecollection": { + "icon": "https://cdn.jsdelivr.net/gh/i3mainz/geopubby@master/public/icons/featurecollection.png"}, + "instance": {"icon": "https://cdn.jsdelivr.net/gh/i3mainz/geopubby@master/public/icons/instance.png"}, + "geoinstance": { + "icon": "https://cdn.jsdelivr.net/gh/i3mainz/geopubby@master/public/icons/geoinstance.png"} + }, + "core": {"themes": {"responsive": True}, "check_callback": True, "data": []}} + tree["@context"] = ldcontext["@context"] + result = [] + ress = {} + for res in results: + # print(res) + if "_:" not in str(res["subject"]) and str(res["subject"]).startswith("http"): + if "_:" not in str(res["supertype"]) and str(res["supertype"]).startswith("http"): + ress[str(res["subject"])] = {"super": res["supertype"], "label": res["label"]} + else: + ress[str(res["subject"])] = {"super": None, "label": res["label"]} + # print(ress) + for cls in ress: + for obj in graph.subjects(URIRef(typeproperty), URIRef(cls), True): + res = DocUtils.replaceNameSpacesInLabel(prefixes, str(obj)) + if str(obj) in uritolabel: + restext = uritolabel[str(obj)]["label"] + " (" + DocUtils.shortenURI(str(obj)) + ")" + if res != None: + restext = uritolabel[str(obj)]["label"] + " (" + res["uri"] + ")" + else: + restext = DocUtils.shortenURI(str(obj)) + if res != None: + restext += " (" + res["uri"] + ")" + if str(obj) not in SPARQLUtils.collectionclasses: + result.append({"id": str(obj), "parent": cls, "type": "instance", "text": restext, "data": {}}) + else: + result.append({"id": str(obj), "parent": cls, "type": "class", "text": restext, "data": {}}) + if str(obj) not in uritotreeitem: + uritotreeitem[str(obj)] = [] + uritotreeitem[str(obj)].append(result[-1]) + # classidset.add(str(obj)) + res = DocUtils.replaceNameSpacesInLabel(prefixes, str(cls)) + if ress[cls]["super"] is None: + restext = DocUtils.shortenURI(str(cls)) + if res is not None: + restext += " (" + res["uri"] + ")" + if cls not in uritotreeitem: + result.append({"id": cls, "parent": "#", "type": "class", "text": restext, "data": {}}) + uritotreeitem[str(cls)] = [] + uritotreeitem[str(cls)].append(result[-1]) + else: + if "label" in cls and ress[cls]["label"] is not None: + restext = ress[cls]["label"] + " (" + DocUtils.shortenURI(str(cls)) + ")" + if res is not None: + restext = ress[cls]["label"] + " (" + res["uri"] + ")" + else: + restext = DocUtils.shortenURI(str(cls)) + if res is not None: + restext += " (" + res["uri"] + ")" + if cls not in uritotreeitem: + result.append({"id": cls, "parent": ress[cls]["super"], "type": "class", "text": restext, "data": {}}) + if str(cls) not in uritotreeitem: + uritotreeitem[str(cls)] = [] + uritotreeitem[str(cls)].append(result[-1]) + else: + uritotreeitem[cls][-1]["parent"] = ress[cls]["super"] + if str(ress[cls]["super"]) not in uritotreeitem: + uritotreeitem[str(ress[cls]["super"])] = [] + clsres = DocUtils.replaceNameSpacesInLabel(prefixes, str(ress[cls]["super"])) + if clsres is not None: + theitem = {"id": str(ress[cls]["super"]), "parent": "#", "type": "class", + "text": DocUtils.shortenURI(str(ress[cls]["super"])) + " (" + clsres["uri"] + ")", + "data": {}} + else: + theitem = {"id": str(ress[cls]["super"]), "parent": "#", "type": "class","text": DocUtils.shortenURI(str(ress[cls]["super"])), "data": {}} + uritotreeitem[str(ress[cls]["super"])].append(theitem) + result.append(theitem) + classidset.add(str(ress[cls]["super"])) + classidset.add(str(cls)) + tree["core"]["data"] = result + return tree + + @staticmethod + def assignGeoClassesToTree(tree): + classlist = {} + for item in tree["core"]["data"]: + if item["type"] == "class": + classlist[item["id"]] = {"items": 0, "geoitems": 0, "item": item} + for item in tree["core"]["data"]: + if item["type"] == "instance" and item["parent"] in classlist: + classlist[item["parent"]]["items"] += 1 + elif (item["type"] == "geoinstance" or item["type"] == "featurecollection" or item[ + "type"] == "geocollection") and item["parent"] in classlist: + classlist[item["parent"]]["items"] += 1 + classlist[item["parent"]]["geoitems"] += 1 + for item in classlist: + if classlist[item]["items"] > 0: + if classlist[item]["item"]["text"].endswith("]"): + classlist[item]["item"]["text"] = classlist[item]["item"]["text"][ + 0:classlist[item]["item"]["text"].rfind("[") - 1] + " [" + str( + classlist[item]["items"]) + "]" + else: + classlist[item]["item"]["text"] = classlist[item]["item"]["text"] + " [" + str( + classlist[item]["items"]) + "]" + if item in SPARQLUtils.collectionclasses: + classlist[item]["item"]["type"] = "collectionclass" + elif classlist[item]["items"] == classlist[item]["geoitems"] and classlist[item]["items"] > 0 and \ + classlist[item]["geoitems"] > 0: + classlist[item]["item"]["type"] = "geoclass" + elif classlist[item]["items"] > classlist[item]["geoitems"] and classlist[item]["geoitems"] > 0: + classlist[item]["item"]["type"] = "halfgeoclass" + else: + classlist[item]["item"]["type"] = "class" + return classlist + + @staticmethod + def checkGeoInstanceAssignment(uritotreeitem): + for uri in uritotreeitem: + if len(uritotreeitem[uri]) > 1: + thetype = "instance" + counter = 0 + if uritotreeitem[uri] != None: + for item in uritotreeitem[uri]: + if item["type"] != "instance" or item["type"] != "class": + thetype = item["type"] + if item["type"] != "class": + item["id"] = item["id"] + "_suniv" + str(counter) + "_" + counter += 1 + if thetype != "instance" or thetype != "class": + for item in uritotreeitem[uri]: + item["type"] = thetype \ No newline at end of file diff --git a/util/doc/docconfig.py b/util/doc/docconfig.py index 5b6b1e657..cc3aeda88 100644 --- a/util/doc/docconfig.py +++ b/util/doc/docconfig.py @@ -214,8 +214,8 @@ class DocConfig: PREFIX rdfs: \n SELECT DISTINCT ?subject ?label ?supertype\n WHERE {\n - { ?individual rdf:type ?subject . } UNION { ?subject rdf:type owl:Class . } UNION { ?subject rdf:type rdfs:Class . } .\n - OPTIONAL { ?subject rdfs:subClassOf ?supertype } .\n + { ?individual %%typeproperty%% ?subject . } UNION { ?subject %%typeproperty%% owl:Class . } UNION { ?subject %%typeproperty%% rdfs:Class . } .\n + OPTIONAL { ?subject %%subclassproperty%% ?supertype } .\n OPTIONAL { ?subject rdfs:label ?label. filter(langMatches(lang(?label),\"en\")) } OPTIONAL { ?subject rdfs:label ?label }.\n FILTER (\n diff --git a/util/doc/docdefaults.py b/util/doc/docdefaults.py index 173b408bd..62e78e49f 100644 --- a/util/doc/docdefaults.py +++ b/util/doc/docdefaults.py @@ -1,7 +1,7 @@ class DocDefaults: templates = { "epsgdefs": "var epsgdefs={}", - "startscripts": """var namespaces={"rdf":"http://www.w3.org/1999/02/22-rdf-syntax-ns#","xsd":"http://www.w3.org/2001/XMLSchema#","geo":"http://www.opengis.net/ont/geosparql#","rdfs":"http://www.w3.org/2000/01/rdf-schema#","owl":"http://www.w3.org/2002/07/owl#","dc":"http://purl.org/dc/terms/","skos":"http://www.w3.org/2004/02/skos/core#"} + "startscripts": """var namespaces={"data":"http://www.w3.org/1999/02/22-rdf-syntax-ns#","xsd":"http://www.w3.org/2001/XMLSchema#","geo":"http://www.opengis.net/ont/geosparql#","rdfs":"http://www.w3.org/2000/01/rdf-schema#","owl":"http://www.w3.org/2002/07/owl#","dc":"http://purl.org/dc/terms/","skos":"http://www.w3.org/2004/02/skos/core#"} var annotationnamespaces=["http://www.w3.org/2004/02/skos/core#","http://www.w3.org/2000/01/rdf-schema#","http://purl.org/dc/terms/"] var indexpage=false var rangesByAttribute={} @@ -109,9 +109,9 @@ class DocDefaults: } function testRDFLibParsing(cururl){ - var store = $rdf.graph() + var store = $data.graph() var timeout = 5000 // 5000 ms timeout - var fetcher = new $rdf.Fetcher(store, timeout) + var fetcher = new $data.Fetcher(store, timeout) fetcher.nowOrWhenFetched(cururl, function(ok, body, response) { if (!ok) { diff --git a/util/doc/docutils.py b/util/doc/docutils.py index 5d78ea091..e9c61c177 100644 --- a/util/doc/docutils.py +++ b/util/doc/docutils.py @@ -1,10 +1,33 @@ from ..sparqlutils import SPARQLUtils import os +import re +import shutil +import urllib.request from rdflib import URIRef +from qgis.core import Qgis, QgsMessageLog + +from .docconfig import DocConfig class DocUtils: + @staticmethod + def zero_div(x,y): + return y and x/y or 0 + + + @staticmethod + def replaceStandardVariables(template, subject, checkdepth, indexpage, pubconfig): + template = template.replace("{{indexpage}}", str(indexpage)).replace("{{subject}}", str(subject)).replace( + "{{relativedepth}}", str(checkdepth)) \ + .replace("{{versionurl}}", DocConfig.versionurl).replace("{{version}}", DocConfig.version).replace( + "{{deploypath}}", pubconfig["deploypath"]) \ + .replace("{{publishingorg}}", pubconfig["publishingorg"]).replace("{{publisher}}", + pubconfig["publisher"]).replace( + "{{datasettitle}}", pubconfig["datasettitle"]) \ + .replace("{{logo}}", pubconfig["logoname"]) + return template + @staticmethod def checkDepthFromPath(savepath,baseurl,subject): if savepath.endswith("/"): @@ -61,7 +84,7 @@ def generateRelativeSymlink(linkpath, targetpath, outpath, items=False): return targetrellink.replace("//", "/") @staticmethod - def getLabelForObject(obj,graph,labellang=None): + def getLabelForObject(obj,graph,prefixes=None,labellang=None): label="" onelabel=DocUtils.shortenURI(str(obj)) for tup in graph.predicate_objects(obj): @@ -70,8 +93,16 @@ def getLabelForObject(obj,graph,labellang=None): if tup[1].language==labellang: label=str(tup[1]) onelabel=str(tup[1]) - if label=="" and onelabel!=None: - label=onelabel + if label=="" and onelabel!=None and onelabel!="": + if prefixes!=None: + res = DocUtils.replaceNameSpacesInLabel(prefixes, str(obj)) + label=res["uri"] + else: + label = onelabel + elif label=="" and (onelabel==None or onelabel=="") and prefixes!=None: + res = DocUtils.replaceNameSpacesInLabel(prefixes, str(obj)) + if res!=None: + label=res["uri"] return label @staticmethod @@ -106,11 +137,15 @@ def processSubjectPath(outpath,paths,path,graph): @staticmethod def replaceNameSpacesInLabel(prefixes,uri): - for ns in prefixes["reversed"]: - if ns in uri: - return {"uri": str(prefixes["reversed"][ns]) + ":" + str(uri.replace(ns, "")), - "ns": prefixes["reversed"][ns]} - return None + nsuri=DocUtils.shortenURI(uri,True) + if prefixes!=None and nsuri in prefixes["reversed"]: + if nsuri==uri and nsuri in prefixes["nstolabel"]: + return {"uri": prefixes["nstolabel"][nsuri]+" ("+str(prefixes["reversed"][nsuri])+":)", + "ns": prefixes["reversed"][nsuri]} + else: + return {"uri": str(prefixes["reversed"][nsuri]) + ":" + str(uri.replace(nsuri, "")), + "ns": prefixes["reversed"][nsuri]} + return {"uri": DocUtils.shortenURI(uri),"ns": ""} @staticmethod def shortenURI(uri,ns=False): @@ -131,4 +166,84 @@ def generateRelativePathFromGivenDepth(checkdepth): rellink = "" for i in range(0, checkdepth): rellink = "../" + rellink - return rellink \ No newline at end of file + return rellink + + @staticmethod + def createOfflineCompatibleVersion(outpath, myhtmltemplate, templatepath, templatename): + if not os.path.isdir(outpath): + os.mkdir(outpath) + if not os.path.isdir(outpath + "/js"): + os.mkdir(outpath + "/js") + if not os.path.isdir(outpath + "/css"): + os.mkdir(outpath + "/css") + matched = re.findall(r'src="(http.*)"', myhtmltemplate) + for match in matched: + # download the library + if "" in match: + for m in match.split(">" in match: - for m in match.split(">" - indexhtml+=templates["footer"].replace("{{license}}",curlicense).replace("{{exports}}",templates["nongeoexports"]).replace("{{bibtex}}","") - #QgsMessageLog.logMessage(path) - indpcounter+=1 - with open(path + "index.html", 'w', encoding='utf-8') as f: - f.write(indexhtml) - f.close() - sparqlhtml = templates["htmltemplate"].replace("{{indexpage}}", "false").replace("{{iconprefixx}}", ( - relpath + "icons/" if self.offlinecompat else "")).replace("{{deploypath}}", self.deploypath).replace( - "{{datasettitle}}", self.datasettitle).replace("{{logo}}", "").replace("{{baseurl}}", - prefixnamespace).replace( - "{{relativedepth}}", "0").replace("{{proprelationpath}}", "proprelations.js").replace("{{relativepath}}", "").replace("{{toptitle}}", - "SPARQL Query Editor").replace( - "{{title}}", "SPARQL Query Editor").replace("{{startscriptpath}}", scriptlink).replace("{{stylepath}}", - stylelink).replace( - "{{vowlpath}}", vowllink) \ - .replace("{{classtreefolderpath}}", classtreelink).replace("{{baseurlhtml}}", "").replace("{{subject}}", + if self.htmlexporter.has3d: + if not os.path.exists(outpath + "/js"): + os.makedirs(outpath + "/js") + with open(outpath + "/js/corto.em.js", 'w', encoding='utf-8') as f: + f.write(templates["corto.em"]) + f.close() + with open(outpath + "/js/nexus.js", 'w', encoding='utf-8') as f: + f.write(templates["nexus"]) + f.close() + if self.pubconfig["apis"]["iiif"]: + IIIFAPIExporter.generateIIIFAnnotations(outpath,imagetoURI) + if self.pubconfig["createIndexPages"]: + IndexViewPage.createIndexPages(self.pubconfig, templates, self.pubconfig["apis"], paths, subjectstorender, + uritotreeitem, voidds, tree, classlist, self.graph, self.voidstatshtml, + curlicense) + if "sparqltemplate" in templates: + sparqlhtml = DocUtils.replaceStandardVariables(templates["htmltemplate"], "", "0", "false", self.pubconfig) + sparqlhtml = sparqlhtml.replace("{{iconprefixx}}", + ("icons/" if self.pubconfig["offlinecompat"] else "")).replace( + "{{baseurl}}", prefixnamespace).replace("{{relativedepth}}", "0").replace("{{relativepath}}", + ".").replace("{{toptitle}}", + "SPARQL Query Editor").replace( + "{{title}}", "SPARQL Query Editor").replace("{{startscriptpath}}", "startscripts.js").replace( + "{{stylepath}}", "style.css") \ + .replace("{{classtreefolderpath}}", self.pubconfig["corpusid"] + "_classtree.js").replace( + "{{baseurlhtml}}", "").replace( + "{{nonnslink}}", "").replace("{{scriptfolderpath}}", self.pubconfig["corpusid"] + "_search.js").replace( + "{{exports}}", + templates[ + "nongeoexports"]).replace( + "{{versionurl}}", DocConfig.versionurl).replace("{{version}}", DocConfig.version).replace("{{bibtex}}", "").replace( - "{{nonnslink}}", "").replace("{{scriptfolderpath}}", sfilelink).replace("{{exports}}", - templates["nongeoexports"]).replace( - "{{versionurl}}", DocConfig.versionurl).replace("{{version}}", DocConfig.version).replace("{{bibtex}}", "") + "{{proprelationpath}}", "proprelations.js") sparqlhtml += templates["sparqltemplate"].replace("{{relativepath}}","") - sparqlhtml += templates["footer"].replace("{{license}}", curlicense).replace("{{exports}}", templates["nongeoexports"]).replace( + tempfoot = templates["footer"].replace("{{license}}", curlicense).replace("{{exports}}", templates["nongeoexports"]).replace( "{{bibtex}}", "") + tempfoot = DocUtils.conditionalArrayReplace(tempfoot, [True, self.pubconfig["apis"]["ogcapifeatures"], self.pubconfig["apis"]["iiif"], self.pubconfig["apis"]["ckan"]], + [ + "[SPARQL] ", + "[OGC API Features] ", + "[IIIF] ", + "[CKAN]" + ], "{{apis}}") + sparqlhtml += tempfoot with open(outpath + "sparql.html", 'w', encoding='utf-8') as f: f.write(sparqlhtml) f.close() + relpath = DocUtils.generateRelativePathFromGivenDepth(0) if len(iiifmanifestpaths["default"])>0: - IIIFAPIExporter.generateIIIFCollections(self.outpath,self.deploypath,iiifmanifestpaths["default"],prefixnamespace) - IIIFAPIExporter.generateImageGrid(self.deploypath, iiifmanifestpaths["default"], templates["imagegrid"], outpath+"imagegrid.html") - if len(featurecollectionspaths)>0 and self.ckan: - CKANExporter.generateCKANCollection(outpath,featurecollectionspaths) - if len(featurecollectionspaths)>0: + IIIFAPIExporter.generateIIIFCollections(self.pubconfig["outpath"],self.pubconfig["deploypath"],iiifmanifestpaths["default"],prefixnamespace) + IIIFAPIExporter.generateImageGrid(self.pubconfig["deploypath"], iiifmanifestpaths["default"], templates["imagegrid"], outpath+"imagegrid.html") + if len(featurecollectionspaths)>0 and self.pubconfig["apis"]["ckan"]: + CKANExporter.generateCKANCollection(outpath, self.pubconfig["deploypath"], + self.htmlexporter.featurecollectionspaths, tree["core"]["data"], + self.pubconfig["license"]) + if self.pubconfig["apis"]["solidexport"]: + SolidExporter.createSolidSettings(self.graph, outpath, self.pubconfig["deploypath"], self.pubconfig["publisher"], self.pubconfig["datasettitle"], + tree["core"]["data"]) + if len(self.htmlexporter.featurecollectionspaths) > 0: relpath=DocUtils.generateRelativePathFromGivenDepth(0) - indexhtml = templates["htmltemplate"].replace("{{iconprefixx}}",(relpath+"icons/" if self.offlinecompat else "")).replace("{{logo}}",self.logoname).replace("{{relativepath}}",relpath).replace("{{relativedepth}}","0").replace("{{baseurl}}", prefixnamespace).replace("{{toptitle}}","Feature Collection Overview").replace("{{title}}","Feature Collection Overview").replace("{{startscriptpath}}", "startscripts.js").replace("{{stylepath}}", "style.css").replace("{{epsgdefspath}}", "epsgdefs.js")\ - .replace("{{classtreefolderpath}}",corpusid + "_classtree.js").replace("{{baseurlhtml}}", "").replace("{{scriptfolderpath}}", corpusid + '_search.js').replace("{{exports}}",templates["nongeoexports"]) + indexhtml = templates["htmltemplate"].replace("{{iconprefixx}}",(relpath+"icons/" if self.pubconfig["offlinecompat"] else "")).replace("{{logo}}",self.pubconfig["logoname"]).replace("{{relativepath}}",relpath).replace("{{relativedepth}}","0").replace("{{baseurl}}", prefixnamespace).replace("{{toptitle}}","Feature Collection Overview").replace("{{title}}","Feature Collection Overview").replace("{{startscriptpath}}", "startscripts.js").replace("{{stylepath}}", "style.css").replace("{{epsgdefspath}}", "epsgdefs.js")\ + .replace("{{classtreefolderpath}}",self.pubconfig["corpusid"] + "_classtree.js").replace("{{baseurlhtml}}", "").replace("{{scriptfolderpath}}", self.pubconfig["corpusid"] + '_search.js').replace("{{exports}}",templates["nongeoexports"]) indexhtml = indexhtml.replace("{{indexpage}}", "true") - OGCAPIFeaturesExporter.generateOGCAPIFeaturesPages(outpath,self.deploypath, featurecollectionspaths, prefixnamespace, self.ogcapifeatures, + OGCAPIFeaturesExporter.generateOGCAPIFeaturesPages(outpath,self.pubconfig["deploypath"], featurecollectionspaths, prefixnamespace, self.pubconfig["apis"]["ogcapifeatures"], True) indexhtml += "

This page shows feature collections present in the linked open data export

" indexhtml+="" indexhtml+=templates["maptemplate"].replace("var ajax=true","var ajax=false").replace("var featurecolls = {{myfeature}}","").replace("{{relativepath}}",DocUtils.generateRelativePathFromGivenDepth(0)).replace("{{baselayers}}",json.dumps(self.baselayers).replace("{{epsgdefspath}}", "epsgdefs.js").replace("{{dateatt}}", "")) - indexhtml += templates["footer"].replace("{{license}}", curlicense).replace("{{exports}}", templates["nongeoexports"]).replace("{{bibtex}}","") + tempfoot = templates["footer"].replace("{{license}}", curlicense).replace("{{exports}}", templates["nongeoexports"]).replace("{{bibtex}}","") + tempfoot = DocUtils.conditionalArrayReplace(tempfoot, [True, self.pubconfig["apis"]["ogcapifeatures"], self.pubconfig["apis"]["iiif"], self.pubconfig["apis"]["ckan"]], + [ + "[SPARQL] ", + "[OGC API Features] ", + "[IIIF] ", + "[CKAN]" + ], "{{apis}}") + indexhtml+=tempfoot with open(outpath + "featurecollections.html", 'w', encoding='utf-8') as f: f.write(indexhtml) f.close() + return subjectstorender - def getPropertyRelations(self, graph, outpath): - predicates = {} - predicatecounter = 0 - predicatelength = 0 - predicateClasses = 0 - objects = set() - for pred in graph.predicates(None, None, True): - predicates[pred] = {"from": set(), "to": set(), "triples": 0} - for tup in graph.subject_objects(pred): - if str(tup[0]) == "http://www.w3.org/1999/02/22-rdf-syntax-ns#type": - predicateClasses += 1 - for item in graph.objects(tup[0], URIRef(self.typeproperty), True): - predicates[pred]["from"].add(item) - for item in graph.objects(tup[1], URIRef(self.typeproperty), True): - predicates[pred]["to"].add(item) - objects.add(str(tup[1])) - predicates[pred]["triples"] += 1 - predicates[pred]["from"] = list(predicates[pred]["from"]) - predicates[pred]["to"] = list(predicates[pred]["to"]) - predicatecounter += 1 - predicatelength += len(str(pred)) - if self.createVOWL: - OWL2VOWL.convertOWL2MiniVOWL(graph, outpath, "minivowl_result.js", predicates) - with open(outpath + "proprelations.js", 'w', encoding='utf-8') as f: - f.write("var proprelations=" + json.dumps(predicates)) - f.close() - return {"preds": predicatecounter, "avgpredlen": str(int(predicatelength / predicatecounter)), - "predclasses": predicateClasses, "objs": len(objects), "predmap": predicates} - - def createCollections(self,graph,namespace): - classToInstances={} - classToGeoColl = {} - classToFColl = {} - for tup in graph.subject_objects(URIRef(self.typeproperty)): - if namespace in str(tup[0]): - if str(tup[1]) not in classToInstances: - classToInstances[str(tup[1])]=set() - classToFColl[str(tup[1])]=0 - classToGeoColl[str(tup[1])] = 0 - classToInstances[str(tup[1])].add(str(tup[0])) - isgeo=False - isfeature = False - for geotup in graph.predicate_objects(tup[0]): - if str(geotup[0]) in SPARQLUtils.geopointerproperties: - isfeature=True - elif str(geotup[0]) in SPARQLUtils.geoproperties: - isgeo=True - if isgeo: - classToGeoColl[str(tup[1])]+=1 - if isfeature: - classToFColl[str(tup[1])]+=1 - for cls in classToInstances: - colluri=namespace+DocUtils.shortenURI(cls)+"_collection" - collrelprop="http://www.w3.org/2000/01/rdf-schema#member" - if classToFColl[cls]==len(classToInstances[cls]): - graph.add((URIRef("http://www.opengis.net/ont/geosparql#SpatialObjectCollection"),URIRef("http://www.w3.org/2000/01/rdf-schema#subClassOf"),URIRef("http://www.w3.org/2004/02/skos/core#Collection"))) - graph.add((URIRef("http://www.opengis.net/ont/geosparql#FeatureCollection"), URIRef("http://www.w3.org/2000/01/rdf-schema#subClassOf"),URIRef("http://www.opengis.net/ont/geosparql#SpatialObjectCollection"))) - graph.add((URIRef(colluri), URIRef(self.typeproperty),URIRef("http://www.opengis.net/ont/geosparql#FeatureCollection"))) - elif classToGeoColl[cls]==len(classToInstances[cls]): - graph.add((URIRef("http://www.opengis.net/ont/geosparql#SpatialObjectCollection"),URIRef("http://www.w3.org/2000/01/rdf-schema#subClassOf"),URIRef("http://www.w3.org/2004/02/skos/core#Collection"))) - graph.add((URIRef("http://www.opengis.net/ont/geosparql#GeometryCollection"), URIRef("http://www.w3.org/2000/01/rdf-schema#subClassOf"),URIRef("http://www.opengis.net/ont/geosparql#SpatialObjectCollection"))) - graph.add((URIRef(colluri), URIRef(self.typeproperty),URIRef("http://www.opengis.net/ont/geosparql#GeometryCollection"))) - elif cls in DocConfig.classToCollectionClass: - if "super" in DocConfig.classToCollectionClass[cls]: - graph.add((URIRef(DocConfig.classToCollectionClass[cls]["class"]), - URIRef("http://www.w3.org/2000/01/rdf-schema#subClassOf"), - URIRef(DocConfig.classToCollectionClass[cls]["super"]))) - graph.add((URIRef(DocConfig.classToCollectionClass[cls]["super"]), - URIRef("http://www.w3.org/2000/01/rdf-schema#subClassOf"), - URIRef("http://www.w3.org/2004/02/skos/core#Collection"))) - else: - graph.add((URIRef(DocConfig.classToCollectionClass[cls]["class"]), - URIRef("http://www.w3.org/2000/01/rdf-schema#subClassOf"), - URIRef("http://www.w3.org/2004/02/skos/core#Collection"))) - graph.add((URIRef(colluri),URIRef(self.typeproperty),URIRef(DocConfig.classToCollectionClass[cls]["class"]))) - collrelprop=DocConfig.classToCollectionClass[cls]["prop"] - else: - graph.add((URIRef(colluri),URIRef(self.typeproperty),URIRef("http://www.w3.org/2004/02/skos/core#Collection"))) - graph.add((URIRef(colluri),URIRef("http://www.w3.org/2000/01/rdf-schema#label"),Literal(str(DocUtils.shortenURI(cls))+" Instances Collection",lang="en"))) - for instance in classToInstances[cls]: - graph.add((URIRef(colluri),URIRef(collrelprop),URIRef(instance))) - return graph - - - def getClassTree(self,graph, uritolabel,classidset,uritotreeitem): - results = graph.query(self.preparedclassquery) - ldcontext={"@context":{ - "@version":1.1, - "foaf":"http://xmlns.com/foaf/0.1/", - "ct":"http://purl.org/vocab/classtree#", - "rdfs":"http://www.w3.org/2000/01/rdf-schema#", - "icon":"foaf:image", - "id":"@id", - "parent":"rdfs:subClassOf", - "halfgeoclass":"ct:HalfGeoClass", - "geoclass":{"@type":"ct:icontype","@id":"ct:GeoClass"}, - "collectionclass":{"@type":"ct:icontype","@id":"ct:CollectionClass"}, - "featurecollectionclass":{"@type":"ct:icontype","@id":"ct:FeatureCollectionClass"}, - "class":"owl:Class", - "instance":"owl:NamedIndividual", - "geoinstance": {"@type":"ct:Icontype","@id":"ct:GeoNamedIndividual"}, - "text":"rdfs:label", - "type":"ct:icontype", - "types":"ct:icontypes", - "core": {"@type":"ct:TreeConfig","@id":"@nest"}, - "data":{"@id":"ct:treeitem","@type":"ct:TreeItem"} - }} - tree = {"plugins": ["defaults","search", "sort", "state", "types", "contextmenu"], "search": {"show_only_matches":True}, - "types": { - "class": {"icon": "https://cdn.jsdelivr.net/gh/i3mainz/geopubby@master/public/icons/class.png"}, - "geoclass": { - "icon": "https://cdn.jsdelivr.net/gh/i3mainz/geopubby@master/public/icons/geoclass.png"}, - "halfgeoclass": { - "icon": "https://cdn.jsdelivr.net/gh/i3mainz/geopubby@master/public/icons/halfgeoclass.png"}, - "collectionclass": { - "icon": "https://cdn.jsdelivr.net/gh/i3mainz/geopubby@master/public/icons/collectionclass.png"}, - "geocollection": { - "icon": "https://cdn.jsdelivr.net/gh/i3mainz/geopubby@master/public/icons/geometrycollection.png"}, - "featurecollection": { - "icon": "https://cdn.jsdelivr.net/gh/i3mainz/geopubby@master/public/icons/featurecollection.png"}, - "instance": { - "icon": "https://cdn.jsdelivr.net/gh/i3mainz/geopubby@master/public/icons/instance.png"}, - "geoinstance": { - "icon": "https://cdn.jsdelivr.net/gh/i3mainz/geopubby@master/public/icons/geoinstance.png"} - }, - "core": {"themes":{"responsive":True},"check_callback": True, "data": []}} - tree["@context"]=ldcontext["@context"] - result = [] - ress = {} - for res in results: - #QgsMessageLog.logMessage(str(res),"OntdocGeneration",Qgis.Info) - if "_:" not in str(res["subject"]) and str(res["subject"]).startswith("http"): - if "_:" not in str(res["supertype"]) and str(res["supertype"]).startswith("http"): - ress[str(res["subject"])] = {"super": res["supertype"], "label": res["label"]} - else: - ress[str(res["subject"])] = {"super": None, "label": res["label"]} - #QgsMessageLog.logMessage(ress) - for cls in ress: - for obj in graph.subjects(URIRef(self.typeproperty), URIRef(cls),True): - res = DocUtils.replaceNameSpacesInLabel(self.prefixes,str(obj)) - if str(obj) in uritolabel: - restext= uritolabel[str(obj)]["label"] + " (" + DocUtils.shortenURI(str(obj)) + ")" - if res!=None: - restext=uritolabel[str(obj)]["label"] + " (" + res["uri"] + ")" - else: - restext= DocUtils.shortenURI(str(obj)) - if res!=None: - restext+= " (" + res["uri"] + ")" - if str(obj) not in SPARQLUtils.collectionclasses: - result.append({"id": str(obj), "parent": cls,"type": "instance","text": restext, "data":{}}) - else: - result.append({"id": str(obj), "parent": cls, "type": "class", "text": restext, "data": {}}) - if str(obj) not in uritotreeitem: - uritotreeitem[str(obj)]=[] - uritotreeitem[str(obj)].append(result[-1]) - #classidset.add(str(obj)) - res = DocUtils.replaceNameSpacesInLabel(self.prefixes,str(cls)) - if ress[cls]["super"] == None: - restext = DocUtils.shortenURI(str(cls)) - if res != None: - restext += " (" + res["uri"] + ")" - if cls not in uritotreeitem: - result.append({"id": cls, "parent": "#","type": "class","text": restext,"data":{}}) - uritotreeitem[str(cls)] = [] - uritotreeitem[str(cls)].append(result[-1]) - else: - if "label" in cls and cls["label"] != None: - restext = ress[cls]["label"] + " (" + DocUtils.shortenURI(str(cls)) + ")" - if res != None: - restext = ress[cls]["label"] + " (" + res["uri"] + ")" - else: - restext = DocUtils.shortenURI(str(cls)) - if res != None: - restext += " (" + res["uri"] + ")" - if cls not in uritotreeitem: - result.append({"id": cls, "parent": ress[cls]["super"],"type": "class","text": restext,"data":{}}) - if str(cls) not in uritotreeitem: - uritotreeitem[str(cls)] = [] - uritotreeitem[str(cls)].append(result[-1]) - else: - uritotreeitem[cls][-1]["parent"]=ress[cls]["super"] - if str(ress[cls]["super"]) not in uritotreeitem: - uritotreeitem[str(ress[cls]["super"])]=[] - clsres = DocUtils.replaceNameSpacesInLabel(self.prefixes,str(ress[cls]["super"])) - if clsres!=None: - theitem = {"id": str(ress[cls]["super"]), "parent": "#", "type": "class", - "text": DocUtils.shortenURI(str(ress[cls]["super"]))+" (" + clsres["uri"] + ")", "data": {}} - else: - theitem={"id": str(ress[cls]["super"]), "parent": "#","type": "class","text": DocUtils.shortenURI(str(ress[cls]["super"])),"data":{}} - uritotreeitem[str(ress[cls]["super"])].append(theitem) - result.append(theitem) - classidset.add(str(ress[cls]["super"])) - classidset.add(str(cls)) - if len(result)==0: - classidset.add("http://www.w3.org/2002/07/owl#Thing") - result.append({"id": "http://www.w3.org/2002/07/owl#Thing", "parent": "#", "type": "class", "text": "Thing (owl:Thing)", "data": {}}) - for obj in graph.subjects(True): - result.append({"id":str(obj) , "parent": "http://www.w3.org/2002/07/owl#Thing", "type": "instance", "text": DocUtils.shortenURI(str(obj)),"data": {}}) - tree["core"]["data"] = result - return tree - - def assignGeoClassesToTree(self,tree): - classlist={} - for item in tree["core"]["data"]: - if item["type"]=="class": - classlist[item["id"]]={"items":0,"geoitems":0,"item":item} - for item in tree["core"]["data"]: - if item["type"]=="instance" and item["parent"] in classlist: - classlist[item["parent"]]["items"]+=1 - elif (item["type"] == "geoinstance" or item["type"]=="featurecollection" or item["type"]=="geocollection") and item["parent"] in classlist: - classlist[item["parent"]]["items"]+=1 - classlist[item["parent"]]["geoitems"]+=1 - for item in classlist: - if classlist[item]["items"]>0: - if classlist[item]["item"]["text"].endswith("]"): - classlist[item]["item"]["text"]=classlist[item]["item"]["text"][0:classlist[item]["item"]["text"].rfind("[")-1]+" ["+str(classlist[item]["items"])+"]" - else: - classlist[item]["item"]["text"]=classlist[item]["item"]["text"]+" ["+str(classlist[item]["items"])+"]" - if item in SPARQLUtils.collectionclasses: - classlist[item]["item"]["type"] = "collectionclass" - elif classlist[item]["items"]==classlist[item]["geoitems"] and classlist[item]["items"]>0 and classlist[item]["geoitems"]>0: - classlist[item]["item"]["type"]="geoclass" - elif classlist[item]["items"]>classlist[item]["geoitems"] and classlist[item]["geoitems"]>0: - classlist[item]["item"]["type"]="halfgeoclass" - else: - classlist[item]["item"]["type"] = "class" - return classlist - - def checkGeoInstanceAssignment(self,uritotreeitem): - for uri in uritotreeitem: - if len(uritotreeitem[uri])>1: - thetype="instance" - counter=0 - if uritotreeitem[uri]!=None: - for item in uritotreeitem[uri]: - if item["type"]!="instance" or item["type"]!="class": - thetype=item["type"] - if item["type"]!="class": - item["id"]=item["id"]+"_suniv"+str(counter)+"_" - counter+=1 - if thetype!="instance" or thetype!="class": - for item in uritotreeitem[uri]: - item["type"]=thetype - - - def resolveTimeObject(self, pred, obj, graph, timeobj): - #QgsMessageLog.logMessage("RESOLVE TIME OBJECT: " +str(pred)+" " + str(obj), "OntdocGeneration", Qgis.Info) - if str(pred) == "http://www.w3.org/2006/time#hasBeginning": - for tobj2 in graph.predicate_objects(obj): - if str(tobj2[0]) in SPARQLUtils.timeproperties: - timeobj["begin"] = tobj2[1] - elif str(pred) == "http://www.w3.org/2006/time#hasEnd": - for tobj2 in graph.predicate_objects(obj): - if str(tobj2[0]) in SPARQLUtils.timeproperties: - timeobj["end"] = tobj2[1] - elif str(pred) == "http://www.w3.org/2006/time#hasTime": - for tobj2 in graph.predicate_objects(obj): - if str(tobj2[0]) in SPARQLUtils.timeproperties: - timeobj["timepoint"] = tobj2[1] - return timeobj - - - def timeObjectToHTML(self, timeobj): - timeres = None - if "begin" in timeobj and "end" in timeobj: - timeres = str(timeobj["begin"]) + " " - if str(timeobj["begin"].datatype) in SPARQLUtils.timeliteraltypes: - timeres += DocUtils.createURILink(self.prefixes,SPARQLUtils.timeliteraltypes[str(timeobj["begin"].datatype)]) - timeres += " - " + str(timeobj["end"]) - if str(timeobj["end"].datatype) in SPARQLUtils.timeliteraltypes: - timeres += DocUtils.createURILink(self.prefixes,SPARQLUtils.timeliteraltypes[str(timeobj["end"].datatype)]) - elif "begin" in timeobj and not "end" in timeobj: - timeres = str(timeobj["begin"]) - if str(timeobj["begin"].datatype) in SPARQLUtils.timeliteraltypes: - timeres += DocUtils.createURILink(self.prefixes,SPARQLUtils.timeliteraltypes[str(timeobj["begin"].datatype)]) - elif "begin" not in timeobj and "end" in timeobj: - timeres = str(timeobj["end"]) - if str(timeobj["end"].datatype) in SPARQLUtils.timeliteraltypes: - timeres += DocUtils.createURILink(self.prefixes,SPARQLUtils.timeliteraltypes[str(timeobj["end"].datatype)]) - elif "timepoint" in timeobj: - timeres = timeobj["timepoint"] - if str(timeobj["timepoint"].datatype) in SPARQLUtils.timeliteraltypes: - timeres += DocUtils.createURILink(self.prefixes,SPARQLUtils.timeliteraltypes[str(timeobj["timepoint"].datatype)]) - return timeres - - def resolveTimeLiterals(self, pred, obj, graph): - timeobj = {} - #QgsMessageLog.logMessage("RESOLVE TIME LITERALS: "+ str(pred)+" " + str(obj), "OntdocGeneration", Qgis.Info) - if isinstance(obj, URIRef) and (str(pred) == "http://www.w3.org/2006/time#hasTime"): - for tobj in graph.predicate_objects(obj): - timeobj = self.resolveTimeObject(tobj[0], tobj[1], graph, timeobj) - elif isinstance(obj, URIRef) and str(pred) in SPARQLUtils.timepointerproperties: - timeobj = self.resolveTimeObject(pred, obj, graph, timeobj) - elif isinstance(obj, Literal): - timeobj = self.resolveTimeObject(pred, obj, graph, timeobj) - return timeobj - - def resolveGeoLiterals(self,pred,object,graph,geojsonrep,nonns,subject=None,treeitem=None,uritotreeitem=None): - #QgsMessageLog.logMessage("RESOLVE " + str(object), "OntdocGeneration", Qgis.Info) - if subject!=None and isinstance(object, Literal) and (str(pred) in SPARQLUtils.geopairproperties): - pairprop = SPARQLUtils.geopairproperties[str(pred)]["pair"] - latorlong = SPARQLUtils.geopairproperties[str(pred)]["islong"] - othervalue = "" - for obj in graph.objects(subject, URIRef(pairprop)): - othervalue = str(obj) - if latorlong: - geojsonrep = {"type": "Point", "coordinates": [float(str(othervalue)), float(str(object))]} - else: - geojsonrep = {"type": "Point", "coordinates": [float(str(object)), float(str(othervalue))]} - elif isinstance(object, Literal) and ( - str(pred) in SPARQLUtils.geoproperties or str(object.datatype) in SPARQLUtils.geoliteraltypes): - geojsonrep = LayerUtils.processLiteral(str(object), str(object.datatype), "") - elif isinstance(object, URIRef) and nonns: - for pobj in graph.predicate_objects(object): - if isinstance(pobj[1], Literal) and ( - str(pobj[0]) in SPARQLUtils.geoproperties or str(pobj[1].datatype) in SPARQLUtils.geoliteraltypes): - geojsonrep = LayerUtils.processLiteral(str(pobj[1]), str(pobj[1].datatype), "") - return geojsonrep - - def searchObjectConnectionsForAggregateData(self,graph,object,pred,geojsonrep,foundmedia,imageannos,textannos,image3dannos,annobodies,timeobj,label,unitlabel,nonns,inverse): - geoprop=False - annosource=None - incollection=False - if pred in SPARQLUtils.geopointerproperties: - geoprop=True - if pred in SPARQLUtils.collectionrelationproperties: - incollection=True - foundval=None - foundunit=None - tempvalprop=None - onelabel=None - bibtex=None - for tup in graph.predicate_objects(object): - if str(tup[0]) in SPARQLUtils.labelproperties: - if isinstance(tup[1],Literal) and tup[1].language==self.labellang: - label=str(tup[1]) - onelabel=str(tup[1]) - if pred=="http://www.w3.org/ns/oa#hasSelector" and tup[0]==URIRef(self.typeproperty) and (tup[1]==URIRef("http://www.w3.org/ns/oa#SvgSelector") or tup[1]==URIRef("http://www.w3.org/ns/oa#WKTSelector")): - for svglit in graph.objects(object,URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#value")): - if "" + str(DocUtils.shortenURI(foundval)) + "" - else: - unitlabel = str(foundval) - if pred=="http://www.w3.org/ns/oa#hasBody": - #print("ADD ANNO BODY: "+str({"value":foundval,"type":"TextualBody","format":"text/plain"})) - annobodies.append({"value":foundval,"type":"TextualBody","format":"text/plain"}) - if annosource != None: - for textanno in textannos: - textanno["src"] = annosource - for imganno in imageannos: - imganno["src"] = annosource - for imganno in image3dannos: - imganno["src"] = annosource - if label=="" and onelabel!=None: - label=onelabel - return {"geojsonrep":geojsonrep,"label":label,"unitlabel":unitlabel,"foundmedia":foundmedia,"imageannos":imageannos,"textannos":textannos,"image3dannos":image3dannos,"annobodies":annobodies,"bibtex":bibtex,"timeobj":timeobj} - - - def createHTMLTableValueEntry(self,subject,pred,object,ttlf,graph,baseurl,checkdepth,geojsonrep,foundmedia,imageannos,textannos,image3dannos,annobodies,dateprops,inverse,nonns): - tablecontents="" - label="" - bibtex = None - timeobj=None - #QgsMessageLog.logMessage("TIME OBJ CREATEHTMLTABLEVALENTRY: "+str(timeobj), "OntdocGeneration", Qgis.Info) - if isinstance(object,URIRef) or isinstance(object,BNode): - if ttlf != None: - ttlf.add((subject,URIRef(pred),object)) - label = "" - unitlabel="" - mydata=self.searchObjectConnectionsForAggregateData(graph,object,pred,geojsonrep,foundmedia,imageannos,textannos,image3dannos,annobodies,timeobj,label,unitlabel,nonns,inverse) - label=mydata["label"] - if label=="": - label=str(DocUtils.shortenURI(str(object))) - geojsonrep=mydata["geojsonrep"] - foundmedia=mydata["foundmedia"] - imageannos=mydata["imageannos"] - textannos=mydata["textannos"] - image3dannos=mydata["image3dannos"] - unitlabel=mydata["unitlabel"] - bibtex=mydata["bibtex"] - timeobj=mydata["timeobj"] - #QgsMessageLog.logMessage("TIME OBJ CREATEHTMLTABLEVALENTRY AFTER AGG: " + str(timeobj), "OntdocGeneration", Qgis.Info) - annobodies=mydata["annobodies"] - if inverse: - rdfares = " about=\"" + str(object) + "\" resource=\"" + str(subject) + "\"" - else: - rdfares = "resource=\"" + str(object) + "\"" - if baseurl in str(object) or isinstance(object,BNode): - rellink = DocUtils.generateRelativeLinkFromGivenDepth(baseurl,checkdepth,str(object),True) - tablecontents += ""+ label + " (" + self.namespaceshort + ":" + str(DocUtils.shortenURI(str(object))) + ")" - if bibtex != None: - tablecontents += "
[BIBTEX]
" + str(bibtex) + "
" - else: - res = DocUtils.replaceNameSpacesInLabel(self.prefixes,str(object)) - if res != None: - tablecontents += "" + label + " (" + res[ - "uri"] + ")" - else: - tablecontents += "" + label + "" - if bibtex!=None: - tablecontents+="
[BIBTEX]
"+str(bibtex)+"
" - if self.generatePagesForNonNS: - rellink = DocUtils.generateRelativeLinkFromGivenDepth(str(baseurl), checkdepth, - str(baseurl) + "nonns_" + DocUtils.shortenURI( - str(object)), False) - tablecontents+=" [x]" - if unitlabel!="": - tablecontents+=" ["+str(unitlabel)+"]" - #QgsMessageLog.logMessage("TIME OBJ CREATEHTMLTABLEVALENTRY AFTER AGG BEFORE TOHTML: " + str(timeobj), "OntdocGeneration", - # Qgis.Info) - if timeobj!=None: - res=str(self.timeObjectToHTML(timeobj)) - if res!="None": - tablecontents+=" ["+str(res)+"]" - dateprops=timeobj - tablecontents+="" - else: - label=str(object) - if ttlf != None: - ttlf.add((subject, URIRef(pred), object)) - if isinstance(object, Literal) and object.datatype != None: - res = DocUtils.replaceNameSpacesInLabel(self.prefixes,str(object.datatype)) - objstring=str(object).replace("<", "<").replace(">", ">") - if str(object.datatype)=="http://www.w3.org/2001/XMLSchema#anyURI": - objstring=""+str(object)+"" - if str(object.datatype) in SPARQLUtils.timeliteraltypes and dateprops!=None and DocUtils.shortenURI(str(pred),True) not in SPARQLUtils.metadatanamespaces and str(pred) not in dateprops: - dateprops.append(str(pred)) - if res != None: - tablecontents += "", ">").replace("\"", "'") + "\" datatype=\"" + str( - object.datatype) + "\">" + self.truncateValue(objstring) + " (" + res["uri"]+ ")" - else: - tablecontents += "", ">").replace("\"", "'") + "\" datatype=\"" + str( - object.datatype) + "\">" + self.truncateValue(objstring) + " (" + DocUtils.shortenURI(str(object.datatype)) + ")" - geojsonrep=self.resolveGeoLiterals(URIRef(pred), object, graph, geojsonrep,nonns,subject) - else: - if object.language != None: - tablecontents += "", ">").replace("\"","'") + "\" datatype=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#langString\" xml:lang=\"" + str(object.language) + "\">" + self.truncateValue(str(object).replace("<", "<").replace(">", ">")) + " (rdf:langString) (iso6391:" + str(object.language) + ")" - else: - tablecontents += self.detectStringLiteralContent(pred,object) - return {"html":tablecontents,"geojson":geojsonrep,"foundmedia":foundmedia,"imageannos":imageannos,"textannos":textannos,"image3dannos":image3dannos,"annobodies":annobodies,"label":label,"timeobj":dateprops} - - def truncateValue(self,value,limit=150): - if len(value)>limit: - return "
"+value[0:limit]+""+str(value[limit:])+"
" - return value - - def detectStringLiteralContent(self,pred,object): - if object.startswith("http://") or object.startswith("https://"): - return "" + str(object)+ " (xsd:string)" - elif object.startswith("www."): - return "http://" + str( - object) + " (xsd:string)" - elif re.search(r'(10[.][0-9]{2,}(?:[.][0-9]+)*/(?:(?![%"#? ])\\S)+)', str(object)): - return "" + str( - object) + " (xsd:anyURI)" - elif re.search(r'[\w.]+\@[\w.]+', object): - return "mailto:" + str( - object) + " (xsd:string)" - return "",">").replace("\"","'") + "\" datatype=\"http://www.w3.org/2001/XMLSchema#string\">" + str(object).replace("<","<").replace(">",">") + " (xsd:string)" - - - def formatPredicate(self,tup,baseurl,checkdepth,tablecontents,graph,reverse): - label=DocUtils.getLabelForObject(URIRef(tup), graph,self.labellang) - tablecontents += "" - if reverse: - tablecontents+="Is " - if baseurl in str(tup): - rellink = DocUtils.generateRelativeLinkFromGivenDepth(baseurl, checkdepth,str(tup),True) - tablecontents += "" + label + "" - else: - res = DocUtils.replaceNameSpacesInLabel(self.prefixes,tup) - if res != None: - tablecontents += "" + label + " (" + res["uri"] + ")" - else: - tablecontents += "" + label + "" - if reverse: - tablecontents+=" of" - tablecontents += "" - return tablecontents def getSubjectPagesForNonGraphURIs(self,uristorender,graph,prefixnamespace,corpusid,outpath,nonnsmap,baseurl,uritotreeitem,labeltouri): #QgsMessageLog.logMessage("Subjectpages " + str(uristorender), "OntdocGeneration", Qgis.Info) @@ -1189,9 +347,9 @@ def getSubjectPagesForNonGraphURIs(self,uristorender,graph,prefixnamespace,corpu if str(tup[0]) in SPARQLUtils.labelproperties: label = str(tup[1]) if uri in uritotreeitem: - res = DocUtils.replaceNameSpacesInLabel(self.prefixes,str(uri)) - label = DocUtils.getLabelForObject(URIRef(str(uri)), graph, self.labellang) - if res != None and label != "": + res = DocUtils.replaceNameSpacesInLabel(self.pubconfig["prefixes"],str(uri)) + label = DocUtils.getLabelForObject(URIRef(str(uri)), graph,self.pubconfig["prefixes"], self.pubconfig["labellang"]) + if res is not None and label != "": uritotreeitem[uri][-1]["text"] = label + " (" + res["uri"] + ")" elif label != "": uritotreeitem[uri][-1]["text"] = label + " (" + DocUtils.shortenURI(uri) + ")" @@ -1202,386 +360,10 @@ def getSubjectPagesForNonGraphURIs(self,uristorender,graph,prefixnamespace,corpu if counter%10==0: self.updateProgressBar(counter,nonnsuris,"NonNS URIs") #QgsMessageLog.logMessage("NonNS Counter " +str(counter)+"/"+str(nonnsuris)+" "+ str(uri), "OntdocGeneration", Qgis.Info) - self.createHTML(outpath+"nonns_"+DocUtils.shortenURI(uri)+".html", None, URIRef(uri), baseurl, graph.subject_predicates(URIRef(uri),True), graph, str(corpusid) + "_search.js", str(corpusid) + "_classtree.js", None, self.license, None, Graph(),uristorender,True,label) + self.htmlexporter.createHTML(outpath+"nonns_"+DocUtils.shortenURI(uri)+".html", None, URIRef(uri), baseurl, graph.subject_predicates(URIRef(uri),True), graph, str(corpusid) + "_search.js", str(corpusid) + "_classtree.js", None, self.pubconfig["license"], None, Graph(),uristorender,True,label) counter+=1 def polygonToPath(self, svg): svg = svg.replace("", " Z\">") return svg.replace("", - "") - - - def getAccessFromBaseURL(self,baseurl,savepath): - #QgsMessageLog.logMessage("Checkdepth: " + baseurl+" "+savepath.replace(baseurl, ""), "OntdocGeneration", Qgis.Info) - return savepath.replace(baseurl, "") - - def createHTML(self,savepath, predobjs, subject, baseurl, subpreds, graph, searchfilename, classtreename,uritotreeitem,curlicense,subjectstorender,postprocessing,nonnsmap=None,nonns=False,foundlabel=""): - tablecontents = "" - metadatatablecontents="" - geojsonrep=None - epsgcode="" - foundmedia={"audio":{},"video":{},"image":{},"mesh":{}} - savepath = savepath.replace("\\", "/") - checkdepth=0 - if not nonns: - checkdepth=DocUtils.checkDepthFromPath(savepath, baseurl, subject) - logo="" - if self.logoname!=None and self.logoname!="": - logo="\"logo\"  " - textannos = [] - foundvals=set() - imageannos=[] - image3dannos=[] - annobodies=[] - predobjmap={} - curtypes=set() - comment={} - parentclass=None - inverse=False - dateprops = [] - timeobj=None - if uritotreeitem!=None and str(subject) in uritotreeitem and uritotreeitem[str(subject)][-1]["parent"].startswith("http"): - parentclass=str(uritotreeitem[str(subject)][-1]["parent"]) - if parentclass not in uritotreeitem: - uritotreeitem[parentclass]=[{"id": parentclass, "parent": "#","type": "class","text": DocUtils.shortenURI(str(parentclass)),"data":{}}] - uritotreeitem[parentclass][-1]["instancecount"]=0 - ttlf = Graph(bind_namespaces="rdflib") - if parentclass!=None: - uritotreeitem[parentclass][-1]["data"]["to"]={} - uritotreeitem[parentclass][-1]["data"]["from"]={} - tablecontentcounter=-1 - metadatatablecontentcounter=-1 - foundtype=False - hasnonns=set() - thetypes = set() - itembibtex = "" - collections=set() - if predobjs!=None: - for tup in sorted(predobjs,key=lambda tup: tup[0]): - if str(tup[0]) not in predobjmap: - predobjmap[str(tup[0])]=[] - predobjmap[str(tup[0])].append(tup[1]) - if parentclass!=None and str(tup[0]) not in uritotreeitem[parentclass][-1]["data"]["to"]: - uritotreeitem[parentclass][-1]["data"]["to"][str(tup[0])]={} - uritotreeitem[parentclass][-1]["data"]["to"][str(tup[0])]["instancecount"] = 0 - if parentclass!=None: - uritotreeitem[parentclass][-1]["data"]["to"][str(tup[0])]["instancecount"]+=1 - uritotreeitem[parentclass][-1]["instancecount"]+=1 - if isinstance(tup[1],URIRef): - foundtype=True - for item in graph.objects(tup[1],URIRef(self.typeproperty)): - thetypes.add(str(item)) - if parentclass!=None: - if item not in uritotreeitem[parentclass][-1]["data"]["to"][str(tup[0])]: - uritotreeitem[parentclass][-1]["data"]["to"][str(tup[0])][item] = 0 - uritotreeitem[parentclass][-1]["data"]["to"][str(tup[0])][item]+=1 - if baseurl not in str(tup[1]) and str(tup[0])!=self.typeproperty: - hasnonns.add(str(tup[1])) - if tup[1] not in nonnsmap: - nonnsmap[str(tup[1])]=set() - nonnsmap[str(tup[1])].add(subject) - if not foundtype: - print("no type") - for tup in predobjmap: - #QgsMessageLog.logMessage(self.shortenURI(str(tup),True),"OntdocGeneration",Qgis.Info) - if self.metadatatable and tup not in SPARQLUtils.labelproperties and DocUtils.shortenURI(str(tup),True) in SPARQLUtils.metadatanamespaces: - thetable=metadatatablecontents - metadatatablecontentcounter+=1 - if metadatatablecontentcounter%2==0: - thetable += "" - else: - thetable += "" - else: - thetable=tablecontents - tablecontentcounter+=1 - if tablecontentcounter%2==0: - thetable += "" - else: - thetable += "" - if str(tup) == self.typeproperty: - for tp in predobjmap[tup]: - thetypes.add(str(tp)) - curtypes.add(str(tp)) - if str(tp) in SPARQLUtils.collectionclasses: - uritotreeitem[str(tp)][-1]["instancecount"] += 1 - collections.add(SPARQLUtils.collectionclasses[str(tp)]) - if str(tp) in DocConfig.bibtextypemappings: - itembibtex="
[BIBTEX]
"+str(BibPage.resolveBibtexReference(graph.predicate_objects(subject),subject,graph))+"
" - thetable=self.formatPredicate(tup, baseurl, checkdepth, thetable, graph,inverse) - if str(tup) in SPARQLUtils.labelproperties: - for lab in predobjmap[tup]: - if lab.language==self.labellang: - foundlabel=lab - if foundlabel=="": - foundlabel = str(predobjmap[tup][0]) - if str(tup) in SPARQLUtils.commentproperties: - comment[str(tup)]=str(predobjmap[tup][0]) - if len(predobjmap[tup]) > 0: - thetable+="" - if len(predobjmap[tup]) > listthreshold: - thetable+="
"+str(len(predobjmap[tup]))+" values" - if len(predobjmap[tup])>1: - thetable+="
    " - labelmap={} - itemcounter = 0 - for item in predobjmap[tup]: - if itemcounter >= maxlistthreshold: - break - if ("POINT" in str(item).upper() or "POLYGON" in str(item).upper() or "LINESTRING" in str(item).upper()) and tup in SPARQLUtils.valueproperties and self.typeproperty in predobjmap and URIRef("http://www.w3.org/ns/oa#WKTSelector") in predobjmap[self.typeproperty]: - image3dannos.append(str(item)) - elif " 1: - labelmap[res["label"]]+="
  • "+str(res["html"])+"
    • " - else: - labelmap[res["label"]] += str(res["html"]) - for lab in sorted(labelmap): - thetable+=str(labelmap[lab]) - if len(predobjmap[tup]) >= maxlistthreshold: - tablecontents += "
  • (...)
    • " - if len(predobjmap[tup])>1: - thetable+="
" - if len(predobjmap[tup]) > listthreshold: - thetable+="
" - thetable+="" - else: - thetable += "" - thetable += "" - if tup not in SPARQLUtils.labelproperties and DocUtils.shortenURI(str(tup), True) in SPARQLUtils.metadatanamespaces: - metadatatablecontents=thetable - else: - tablecontents=thetable - subpredsmap={} - if subpreds!=None: - for tup in sorted(subpreds,key=lambda tup: tup[1]): - if str(tup[1]) not in subpredsmap: - subpredsmap[str(tup[1])]=[] - subpredsmap[str(tup[1])].append(tup[0]) - if parentclass!=None and str(tup[1]) not in uritotreeitem[parentclass][-1]["data"]["from"]: - uritotreeitem[parentclass][-1]["data"]["from"][str(tup[1])]={} - uritotreeitem[parentclass][-1]["data"]["from"][str(tup[1])]["instancecount"] = 0 - if isinstance(tup[0],URIRef): - for item in graph.objects(tup[0],URIRef(self.typeproperty)): - if parentclass!=None: - if item not in uritotreeitem[parentclass][-1]["data"]["from"][str(tup[1])]: - uritotreeitem[parentclass][-1]["data"]["from"][str(tup[1])][item] = 0 - uritotreeitem[parentclass][-1]["data"]["from"][str(tup[1])][item]+=1 - for tup in subpredsmap: - tablecontentcounter+=1 - if tablecontentcounter%2==0: - tablecontents += "" - else: - tablecontents += "" - tablecontents=self.formatPredicate(tup, baseurl, checkdepth, tablecontents, graph,True) - if len(subpredsmap[tup]) > 0: - tablecontents += "" - if len(subpredsmap[tup]) > listthreshold: - tablecontents+="
"+str(len(subpredsmap[tup]))+" values" - if len(subpredsmap[tup]) > 1: - tablecontents += "
    " - labelmap={} - itemcounter = 0 - for item in subpredsmap[tup]: - if itemcounter >= maxlistthreshold: - break - if subjectstorender!=None and item not in subjectstorender and baseurl in str(item): - postprocessing.add((item,URIRef(tup),subject)) - res = self.createHTMLTableValueEntry(subject, tup, item, None, graph, - baseurl, checkdepth, geojsonrep,foundmedia,imageannos,textannos,image3dannos,annobodies,dateprops,True,nonns) - foundmedia = res["foundmedia"] - imageannos=res["imageannos"] - image3dannos=res["image3dannos"] - annobodies=res["annobodies"] - if nonns and str(tup) != self.typeproperty: - hasnonns.add(str(item)) - if nonns: - geojsonrep=res["geojson"] - if res["label"] not in labelmap: - labelmap[res["label"]]="" - if len(subpredsmap[tup]) > 1: - labelmap[res["label"]]+="
  • "+str(res["html"])+"
    • " - else: - labelmap[res["label"]] += str(res["html"]) - for lab in sorted(labelmap): - tablecontents+=str(labelmap[lab]) - if len(subpredsmap[tup]) >= maxlistthreshold: - tablecontents+="
  • (...)
    • " - if len(subpredsmap[tup])>1: - tablecontents+="
" - if len(subpredsmap[tup]) > listthreshold: - tablecontents+="
" - tablecontents += "" - else: - tablecontents += "" - tablecontents += "" - if self.licenseuri!=None: - ttlf.add((subject, URIRef("http://purl.org/dc/elements/1.1/license"), URIRef(self.licenseuri))) - nonnslink="" - if nonns: - completesavepath = savepath - nonnslink = "
This page describes linked instances to the concept " + str(foundlabel) + " (" + str(DocUtils.shortenURI( - subject)) + ") in this knowledge graph. It is defined here
" - else: - completesavepath=savepath + "/index.html" - if not nonns: - ttlf.serialize(savepath + "/index.ttl", encoding="utf-8") - with open(savepath + "/index.json", 'w', encoding='utf-8') as f: - f.write(json.dumps(predobjmap)) - f.close() - with open(completesavepath, 'w', encoding='utf-8') as f: - rellink=DocUtils.generateRelativeLinkFromGivenDepth(baseurl,checkdepth,searchfilename,False) - rellink2 = DocUtils.generateRelativeLinkFromGivenDepth(baseurl,checkdepth,classtreename,False) - rellink3 = DocUtils.generateRelativeLinkFromGivenDepth(baseurl,checkdepth,"style.css",False) - rellink4 = DocUtils.generateRelativeLinkFromGivenDepth(baseurl, checkdepth, "startscripts.js", False) - rellink5 = DocUtils.generateRelativeLinkFromGivenDepth(baseurl, checkdepth, "proprelations.js", False) - epsgdefslink = DocUtils.generateRelativeLinkFromGivenDepth(baseurl, checkdepth, "epsgdefs.js", False) - rellink7 = DocUtils.generateRelativeLinkFromGivenDepth(baseurl, checkdepth, "vowl_result.js", False) - if geojsonrep != None: - myexports=templates["geoexports"] - else: - myexports=templates["nongeoexports"] - itembibtex="" - relpath=DocUtils.generateRelativePathFromGivenDepth(checkdepth) - if foundlabel != "": - f.write(templates["htmltemplate"].replace("{{iconprefixx}}",(relpath+"icons/" if self.offlinecompat else "")).replace("{{deploypath}}",self.deploypath).replace("{{logo}}",logo).replace("{{relativepath}}",relpath).replace("{{baseurl}}",baseurl).replace("{{relativedepth}}",str(checkdepth)).replace("{{prefixpath}}", self.prefixnamespace).replace("{{toptitle}}", foundlabel).replace( - "{{startscriptpath}}", rellink4).replace("{{bibtex}}",itembibtex).replace( - "{{epsgdefspath}}", epsgdefslink).replace("{{versionurl}}",DocConfig.versionurl).replace("{{version}}",DocConfig.version).replace("{{bibtex}}",itembibtex).replace("{{vowlpath}}", rellink7).replace("{{proprelationpath}}", rellink5).replace("{{stylepath}}", rellink3).replace("{{indexpage}}","false").replace("{{title}}", - "" + str(foundlabel) + "").replace( - "{{baseurl}}", baseurl).replace("{{tablecontent}}", tablecontents).replace("{{description}}","").replace( - "{{scriptfolderpath}}", rellink).replace("{{classtreefolderpath}}", rellink2).replace("{{exports}}",myexports).replace("{{nonnslink}}",str(nonnslink)).replace("{{subject}}",str(subject)).replace("{{subjectencoded}}",urllib.parse.quote(str(subject)))) - else: - f.write(templates["htmltemplate"].replace("{{iconprefixx}}",(relpath+"icons/" if self.offlinecompat else "")).replace("{{deploypath}}",self.deploypath).replace("{{logo}}",logo).replace("{{relativepath}}",relpath).replace("{{baseurl}}",baseurl).replace("{{relativedepth}}",str(checkdepth)).replace("{{prefixpath}}", self.prefixnamespace).replace("{{indexpage}}","false").replace("{{toptitle}}", DocUtils.shortenURI(str(subject))).replace( - "{{startscriptpath}}", rellink4).replace("{{bibtex}}",itembibtex).replace( - "{{epsgdefspath}}", epsgdefslink).replace("{{versionurl}}",DocConfig.versionurl).replace("{{version}}",DocConfig.version).replace("{{bibtex}}",itembibtex).replace("{{vowlpath}}", rellink7).replace("{{proprelationpath}}", rellink5).replace("{{stylepath}}", rellink3).replace("{{title}}","" + DocUtils.shortenURI(str(subject)) + "").replace( - "{{baseurl}}", baseurl).replace("{{description}}", "").replace( - "{{scriptfolderpath}}", rellink).replace("{{classtreefolderpath}}", rellink2).replace("{{exports}}",myexports).replace("{{nonnslink}}",str(nonnslink)).replace("{{subject}}",str(subject)).replace("{{subjectencoded}}",urllib.parse.quote(str(subject)))) - for comm in comment: - f.write(templates["htmlcommenttemplate"].replace("{{comment}}", DocUtils.shortenURI(comm) + ":" + comment[comm])) - #for fval in foundvals: - # f.write(templates["htmlcommenttemplate"].replace("{{comment}}", "Value:" + str(fval) + "")) - if len(foundmedia["mesh"])>0 and len(image3dannos)>0: - if self.iiif: - iiifmanifestpaths["default"].append( - IIIFAPIExporter.generateIIIFManifest(graph,self.outpath,self.deploypath, foundmedia["mesh"], image3dannos,annobodies, str(subject), - self.prefixnamespace,imagetoURI, self.imagemetadata,SPARQLUtils.metadatanamespaces,foundlabel,comment,thetypes,predobjmap,"Model")) - for anno in image3dannos: - if ("POINT" in anno.upper() or "POLYGON" in anno.upper() or "LINESTRING" in anno.upper()): - f.write(templates["threejstemplate"].replace("{{wktstring}}",anno).replace("{{meshurls}}",str(list(foundmedia["mesh"])))) - elif len(foundmedia["mesh"])>0 and len(image3dannos)==0: - #QgsMessageLog.logMessage("Found 3D Model: "+str(foundmedia["mesh"])) - if self.iiif: - iiifmanifestpaths["default"].append( - IIIFAPIExporter.generateIIIFManifest(graph,self.outpath, self.deploypath, foundmedia["mesh"], annobodies, image3dannos, str(subject),self.prefixnamespace,imagetoURI, self.imagemetadata,SPARQLUtils.metadatanamespaces,foundlabel,comment,thetypes,predobjmap,"Model")) - for curitem in foundmedia["mesh"]: - format="ply" - if ".nxs" in curitem or ".nxz" in curitem: - format="nexus" - f.write(templates["3dtemplate"].replace("{{meshurl}}",curitem).replace("{{meshformat}}",format)) - break - elif len(foundmedia["mesh"])==0 and len(image3dannos)>0: - for anno in image3dannos: - if ("POINT" in anno.upper() or "POLYGON" in anno.upper() or "LINESTRING" in anno.upper()): - f.write(templates["threejstemplate"].replace("{{wktstring}}",anno).replace("{{meshurls}}","[]")) - carousel="image" - if len(foundmedia["image"])>3: - carousel="carousel-item active" - f.write(templates["imagecarouselheader"]) - if len(imageannos)>0 and len(foundmedia["image"])>0: - if self.iiif: - iiifmanifestpaths["default"].append( - IIIFAPIExporter.generateIIIFManifest(graph,self.outpath,self.deploypath,foundmedia["image"],imageannos,annobodies,str(subject),self.prefixnamespace,imagetoURI, self.imagemetadata,SPARQLUtils.metadatanamespaces,foundlabel,comment,thetypes,predobjmap,"Image")) - for image in foundmedia["image"]: - annostring="" - for anno in imageannos: - annostring+=anno.replace("","") - f.write(templates["imageswithannotemplate"].replace("{{carousel}}",carousel+"\" style=\"position: relative;display: inline-block;").replace("{{image}}",str(image)).replace("{{svganno}}",annostring).replace("{{imagetitle}}",str(image)[0:str(image).rfind('.')])) - if len(foundmedia["image"])>3: - carousel="carousel-item" - elif len(foundmedia["image"])>0: - if self.iiif: - iiifmanifestpaths["default"].append( - IIIFAPIExporter.generateIIIFManifest(graph,self.outpath,self.deploypath,foundmedia["image"],imageannos,annobodies,str(subject),self.prefixnamespace,imagetoURI, self.imagemetadata,SPARQLUtils.metadatanamespaces,foundlabel,comment,thetypes,predobjmap,"Image")) - for image in foundmedia["image"]: - if image=="" in image: - f.write(templates["imagestemplatesvg"].replace("{{carousel}}",carousel).replace("{{image}}", str(image.replace("","")))) - else: - f.write(templates["imagestemplatesvg"].replace("{{carousel}}",carousel).replace("{{image}}",str(image))) - else: - f.write(templates["imagestemplate"].replace("{{carousel}}",carousel).replace("{{image}}",str(image)).replace("{{imagetitle}}",str(image)[0:str(image).rfind('.')])) - if len(foundmedia["image"])>3: - carousel="carousel-item" - if len(foundmedia["image"])>3: - f.write(templates["imagecarouselfooter"]) - if len(textannos) > 0: - TextAnnoPage().generatePageWidget(textannos,f) - if len(foundmedia["audio"]) > 0 and self.iiif: - iiifmanifestpaths["default"].append( - IIIFAPIExporter.generateIIIFManifest(graph,self.outpath,self.deploypath,foundmedia["audio"],None,None,str(subject),self.prefixnamespace,imagetoURI, self.imagemetadata,SPARQLUtils.metadatanamespaces,foundlabel,comment,thetypes,predobjmap,"Audio")) - for audio in foundmedia["audio"]: - f.write(templates["audiotemplate"].replace("{{audio}}",str(audio))) - if len(foundmedia["video"]) > 0 and self.iiif: - iiifmanifestpaths["default"].append( - IIIFAPIExporter.generateIIIFManifest(graph,self.outpath,self.deploypath,foundmedia["video"],None,None,str(subject),self.prefixnamespace,imagetoURI, self.imagemetadata,SPARQLUtils.metadatanamespaces,foundlabel,comment,thetypes,predobjmap,"Video")) - for video in foundmedia["video"]: - f.write(templates["videotemplate"].replace("{{video}}",str(video))) - for type in curtypes: - if type in SPARQLUtils.lexicontypes: - LexiconPage().generatePageWidget(graph,subject,f,{},False) - if type in PersonPage.pageWidgetConstraint(): - PersonPage().generatePageWidget(graph,subject,templates,f,True) - self.processCollectionPages(collections, graph, subject, f) - if geojsonrep != None and "geocollection" not in collections: - self.geocache=GeometryViewPage().generatePageWidget(graph, templates, subject, f, uritotreeitem, geojsonrep, predobjmap, self.geocache, - {"dateprops": dateprops, "timeobj": timeobj, - "epsgcode": epsgcode, "epsgdefslink": epsgdefslink, - "checkdepth": checkdepth, "hasnonnslen": len(hasnonns)}) - elif "geocollection" in collections or nonns: - self.geocache=GeometryViewPage().generateCollectionWidget(graph, templates, subject, f, uritotreeitem, - featurecollectionspaths, - {"completesavepath":completesavepath,"nonns":nonns,"hasnonns":hasnonns, - "foundlabel":foundlabel,"localOptimized": self.localOptimized, - "dateprops": dateprops, "timeobj": timeobj, - "geocache":self.geocache, - "epsgcode": epsgcode, "epsgdefslink": epsgdefslink, - "checkdepth": checkdepth, "hasnonnslen": len(hasnonns)}) - f.write(templates["htmltabletemplate"].replace("{{tablecontent}}", tablecontents)) - if metadatatablecontentcounter>=0: - f.write("
Metadata
") - f.write(templates["htmltabletemplate"].replace("{{tablecontent}}", metadatatablecontents)) - f.write(templates["footer"].replace("{{exports}}",myexports).replace("{{license}}",curlicense).replace("{{bibtex}}",itembibtex)) - f.close() - return [postprocessing,nonnsmap] - - def processCollectionPages(self,pagesmap,graph,subject,f): - if "observationcollection" in pagesmap: - ObservationPage().generateCollectionWidget(graph, templates, subject, f,{}) - if "lexicon" in pagesmap: - LexiconPage().generateCollectionWidget(graph,templates,subject,f,{}) \ No newline at end of file + "") \ No newline at end of file diff --git a/util/doc/templateutils.py b/util/doc/templateutils.py index e752585fb..df463b968 100644 --- a/util/doc/templateutils.py +++ b/util/doc/templateutils.py @@ -63,6 +63,22 @@ def resolveTemplate(templatename,templatepath): if filename.endswith(".html") or filename.endswith(".css"): with open(templatepath+"/"+templatename+"/templates/"+filename, 'r') as f: templates[filename.replace(".html","")] = f.read() + if os.path.exists(templatepath+"/"+templatename+"/templates/js/"): + for filename in os.listdir(templatepath+"/"+templatename+"/js/"): + print("FOUND INCLUDE: "+str(filename)) + if filename.endswith(".js"): + with open(templatepath+"/"+templatename+"/js/"+filename, 'r') as f: + content=f.read() + templates["js"][filename.replace(".js","")] = content + templates[filename.replace(".js", "")] = content + if os.path.exists(templatepath+"/"+templatename+"/css/"): + for filename in os.listdir(templatepath + "/" + templatename + "/css/"): + print("FOUND LAYOUT: " + str(filename)) + if filename.endswith(".css"): + with open(templatepath + "/" + templatename + "/css/" + filename, 'r') as f: + content=f.read() + templates[filename.replace(".css", "")] = content + templates[filename.replace(".css", "")] = content print("Found templates.... "+str(len(templates))) for temp in templates: if temp!="includes" and temp!="layouts": diff --git a/util/export/api/ckanexporter.py b/util/export/api/ckanexporter.py index ef811931b..f223e843d 100644 --- a/util/export/api/ckanexporter.py +++ b/util/export/api/ckanexporter.py @@ -6,26 +6,99 @@ from ...doc.docutils import DocUtils + class CKANExporter: + @staticmethod - def generateCKANCollection(outpath, featurecollectionspaths,version="3"): + def generateCKANCollection(outpath, deploypath, featurecollectionspaths,classtree,license="",version="3"): + ckanapihtml = "SwaggerUI
" + ckanopenapi = {"openapi": "3.0.1","info": {"title": str(deploypath)+" CKAN API", "version": str(version)}, "servers": [{"url": str(deploypath)+"/api/3/"}], "paths": {}} if not os.path.exists(outpath + "/dataset/"): os.makedirs(outpath + "/dataset/") if not os.path.exists(outpath + "/api/"): os.makedirs(outpath + "/api/") + if not os.path.exists(outpath + "/api/action"): + os.makedirs(outpath + "/api/action") if not os.path.exists(outpath + "/api/"+str(version)+"/"): os.makedirs(outpath + "/api/"+str(version)+"/") if not os.path.exists(outpath + "/api/"+str(version)+"/action/"): os.makedirs(outpath + "/api/"+str(version)+"/action/") if not os.path.exists(outpath + "/api/"+str(version)+"/action/group_list/"): os.makedirs(outpath + "/api/"+str(version)+"/action/group_list/") - if not os.path.exists(outpath + "/api/"+str(version)+"/action/action_list/"): - os.makedirs(outpath + "/api/"+str(version)+"/action/action_list/") + if not os.path.exists(outpath + "/api/"+str(version)+"/action/package_list/"): + os.makedirs(outpath + "/api/"+str(version)+"/action/package_list/") if not os.path.exists(outpath + "/api/"+str(version)+"/action/tag_list/"): os.makedirs(outpath + "/api/"+str(version)+"/action/tag_list/") + if not os.path.exists(outpath + "/api/"+str(version)+"/action/package_search"): + p = Path(str(outpath + "/api/"+str(version)+"/action/package_search")) + p.symlink_to("./package_list/") + if not os.path.exists(outpath + "/api/"+str(version)+"/action/group_list?all_fields=true"): + p = Path(outpath + "/api/"+str(version)+"/action/group_list?all_fields=true") + p.symlink_to("./group_list/") + if not os.path.exists(outpath + "/api/action/package_search/"): + p = Path(outpath + "/api/action/package_search/") + p.symlink_to("../"+str(version)+"/action/package_list/") + if not os.path.exists(outpath + "/api/action/group_list/"): + p = Path(outpath + "/api/action/group_list/") + p.symlink_to("../"+str(version)+"/action/group_list/") + if not os.path.exists(outpath + "/api/action/package_list/"): + p = Path(outpath + "/api/action/package_list/") + p.symlink_to("../"+str(version)+"/action/package_list/") + if not os.path.exists(outpath + "/api/action/tag_list/"): + p = Path(outpath + "/api/action/tag_list/") + p.symlink_to("../"+str(version)+"/action/tag_list/") + f = open(outpath + "/api/"+str(version)+"/index.json", "w") + f.write(json.dumps({"version": int(version)})) + f.close() + ckanopenapi["paths"]["/api/"+str(version)+"/action/group_list/"] = {"get": {"tags": ["CKAN"], + "summary": "Retrieves the group list of this CKAN API", + "description": "Retrieves the group list of this CKAN API", + "operationId": "actionapi-group_list", + "parameters": [], "responses": { + "200": {"description": "Success", + "content": {"text/plain": {"example": None}}, + "application/json": {"schema": {"example": None}, "example": None}, + "text/json": {"schema": {"example": None}, "example": None}}}}} + ckanopenapi["paths"]["/api/"+str(version)+"/action/tag_list/"] = {"get": {"tags": ["CKAN"], + "summary": "Retrieves the tag list of this CKAN API", + "description": "Retrieves the tag list of this CKAN API", + "operationId": "actionapi-tag_list", + "parameters": [], "responses": { + "200": {"description": "Success", + "content": {"text/plain": {"example": None}}, + "application/json": {"schema": {"example": None}, "example": None}, + "text/json": {"schema": {"example": None}, "example": None}}}}} + ckanopenapi["paths"]["/api/"+str(version)+"/action/package_list/"] = {"get": {"tags": ["CKAN"], + "summary": "Retrieves the package list of this CKAN API", + "description": "Retrieves the package list of this CKAN API", + "operationId": "actionapi-package_list", + "parameters": [], "responses": { + "200": {"description": "Success", + "content": {"text/plain": {"example": None}}, + "application/json": {"schema": {"example": None}, "example": None}, + "text/json": {"schema": {"example": None}, "example": None}}}}} + f = open(outpath + "/api/"+str(version)+"/api.json", "w") + f.write(json.dumps(ckanopenapi)) + f.close() + f = open(outpath + "/api/"+str(version)+"/index.html", "w") + f.write(ckanapihtml) + f.close() f = open(outpath + "/api/"+str(version)+"/action/group_list/index.json", "w") - f.write(json.dumps({"success": True, "result": []})) + if classtree!=None and len(classtree)>0: + classes=[] + for item in classtree: + if item["type"]=="class" or item["type"]=="geoclass": + theid=DocUtils.shortenURI(item["id"]) + classes.append({"description":theid,"display_name":item["text"],"name":item["text"],"title":item["text"],"type":"group"}) + if not os.path.exists(outpath + "/api/"+str(version)+"/action/group_show?id="+theid): + os.makedirs(outpath + "/api/"+str(version)+"/action/group_show?id="+theid) + groupdesc={"success":True,"result":{"description":theid,"display_name":item["text"],"name":item["text"],"title":item["text"],"type":"group"}} + with open(outpath + "/api/"+str(version)+"/action/group_show?id="+theid+"/index.json", 'w') as fl: + fl.write(json.dumps(groupdesc)) + f.write(json.dumps({"success": True, "result": classes})) + else: + f.write(json.dumps({"success": True, "result": []})) f.close() f = open(outpath + "/api/"+str(version)+"/action/tag_list/index.json", "w") f.write(json.dumps({"success": True, "result": ["ttl", "json", "geojson", "html"]})) @@ -35,23 +108,40 @@ def generateCKANCollection(outpath, featurecollectionspaths,version="3"): op = outpath + "/dataset/" + coll.replace(outpath, "").replace("index.geojson", "") op = op.replace(".geojson", "") op = op.replace("//", "/") + curcollname=coll.replace(outpath, "").replace("index.geojson", "") + if curcollname.endswith("/"): + curcollname=curcollname[0:-1] + curcollname=curcollname.replace(".geojson","") if op.endswith("/"): op = op[0:-1] if not os.path.exists(op): os.makedirs(op) - targetpath = DocUtils.generateRelativeSymlink(coll.replace("//", "/"), str(op + ".json"), - outpath) - p = Path(str(op + ".json").replace("//", "/")) - p.symlink_to(targetpath) - targetpath = DocUtils.generateRelativeSymlink(coll.replace("//", "/"), str(op + ".ttl"), - outpath) - p = Path(str(op + ".ttl").replace("//", "/")) - p.symlink_to(targetpath) - targetpath = DocUtils.generateRelativeSymlink(coll.replace("//", "/"), str(op + ".html"), - outpath) - p = Path(str(op + ".html").replace("//", "/")) - p.symlink_to(targetpath) - colls.append(op[op.rfind('/') + 1:]) - f = open(outpath + "/api/action/"+str(version)+"/action_list/index.json", "w") - f.write(json.dumps({"success": True, "result": colls})) + dataset={"success":True,"result":{"id":curcollname,"type":"dataset","num_resources":3,"title":curcollname,"license_id":license,"license_title":license,"name":curcollname,"notes":"","tags":[],"groups":[],"resources":[{"name":curcollname+" (text/ttl)","format":"TTL","id":curcollname,"package_id":curcollname,"mimetype":"text/ttl","resource_type":"file","url":deploypath+"/dataset/"+curcollname+".ttl","state":"active","url_type":""},{"name":curcollname+" (application/json)","id":curcollname,"format":"JSON","package_id":curcollname,"mimetype":"application/json","resource_type":"file","url":deploypath+"/dataset/"+curcollname+".json","state":"active","url_type":""},{"name":curcollname+" (text/html)","id":curcollname,"format":"HTML","package_id":curcollname,"mimetype":"text/html","resource_type":"file","url":deploypath+"/dataset/"+curcollname+".html","state":"active","url_type":""}]}} + f = open(outpath + "/dataset/"+curcollname+"_", "w") + f.write(json.dumps(dataset)) + f.close() + if not os.path.exists(str(op + ".json").replace("//", "/")): + targetpath = DocUtils.generateRelativeSymlink(coll.replace("//", "/"), str(op + ".json"), + outpath) + p = Path(str(op + ".json").replace("//", "/")) + p.symlink_to(targetpath) + p = Path(outpath + "/api/"+str(version)+"/action/package_show?id="+str(curcollname)+".json") + p.symlink_to("../../"+targetpath) + if not os.path.exists(str(op + ".ttl").replace("//", "/")): + targetpath = DocUtils.generateRelativeSymlink(coll.replace("//", "/"), str(op + ".ttl"), + outpath) + p = Path(str(op + ".ttl").replace("//", "/")) + p.symlink_to(targetpath) + p = Path(outpath + "/api/"+str(version)+"/action/package_show?id="+str(curcollname)+".ttl") + p.symlink_to("../../"+targetpath) + if not os.path.exists(str(op + ".html").replace("//", "/")): + targetpath = DocUtils.generateRelativeSymlink(coll.replace("//", "/"), str(op + ".html"), + outpath) + p = Path(str(op + ".html").replace("//", "/")) + p.symlink_to(targetpath) + p = Path(outpath + "/api/"+str(version)+"/action/package_show?id="+str(curcollname)+".html") + p.symlink_to("../../"+targetpath) + colls.append(dataset["result"]) + f = open(outpath + "/api/"+str(version)+"/action/package_list/index.json", "w") + f.write(json.dumps({"success": True, "result":{"count": len(colls), "results":colls}})) f.close() \ No newline at end of file diff --git a/util/export/api/iiifexporter.py b/util/export/api/iiifexporter.py index 83991e27b..6ed802d97 100644 --- a/util/export/api/iiifexporter.py +++ b/util/export/api/iiifexporter.py @@ -15,7 +15,7 @@ class IIIFAPIExporter: @staticmethod def generateIIIFAnnotations(outpath,imagetoURI): for imgpath in imagetoURI: - print("Generate IIIF Annotations for " + str(imgpath) + " with " + str(imagetoURI[imgpath])) + #print("Generate IIIF Annotations for " + str(imgpath) + " with " + str(imagetoURI[imgpath])) if "uri" in imagetoURI[imgpath]: for ur in imagetoURI[imgpath]["uri"]: # print(ur) @@ -39,8 +39,7 @@ def generateIIIFAnnotations(outpath,imagetoURI): f.close() @staticmethod - def generateIIIFManifest(g, outpath, deploypath, imgpaths, annos, annobodies, curind, prefixnamespace, imagetoURI, - imagemetadata, metadatanamespaces, label="", + def generateIIIFManifest(g, outpath, deploypath, imgpaths, annos, annobodies, curind, prefixnamespace, imagetoURI, imagemetadata,metadatanamespaces, label="", summary="", thetypes=None, predobjmap=None, maintype="Image"): #print("GENERATE IIIF Manifest for " + str(outpath) + " " + str(curind) + " " + str(label) + " " + str( # summary) + " " + str(annobodies)) @@ -148,27 +147,78 @@ def generateIIIFManifest(g, outpath, deploypath, imgpaths, annos, annobodies, cu break if besttype == "" and len(thetypes) > 0: besttype = next(iter(thetypes)) - return {"url": outpath + "/iiif/mf/" + DocUtils.shortenURI(curind) + "/manifest.json", "label": str(label), + return {"url": outpath + "/iiif/mf/" + DocUtils.shortenURI(curind) + "/manifest.json", "imgpath": list(imgpaths.keys()), "label": str(label), "class": besttype,"ind":curind} @staticmethod - def generateImageGrid(deploypath,imagespaths,imagegridtemplate,targetfile=None): + def generateImageGrid(outpath,deploypath,imagespaths,imagegridtemplate,headertemplate,footertemplate,targetfile=None): categories=set() imghtml="" - for imgpath in sorted(imagespaths, key=lambda k: k['label'], reverse=False): + for imgpath in imagespaths: + #print("IMAGEPATH: "+str(imgpath)) categories.add(DocUtils.shortenURI(imgpath["class"])) - imghtml+="
  • " - imghtml+="\"{{textName}}\"/
    " - imghtml+="
    "+str(imgpath["label"])+"
    • " + for imgp in imgpath["imgpath"]: + imghtml+="
  • " + imghtml+="\""+str(imgpath["label"])+"\"/
    " + imghtml+="
    " + if imgpath["label"]!="": + imghtml+=str(imgpath["label"])+"
    • " + else: + imghtml += DocUtils.shortenURI(imgpath["url"].replace("/manifest.json", "")) + "" if targetfile!=None: f = open(targetfile, "w") - f.write(imagegridtemplate.replace("{{imagecontainers}}",imghtml).replace("{{categories}}",str(categories))) + f.write(headertemplate) + f.write(imagegridtemplate.replace("{{imagecontainers}}",imghtml).replace("{{categories}}",str(categories).replace("{","").replace("}",""))) + f.write(footertemplate) f.close() else: - return imagegridtemplate.replace("{{imagecontainers}}",imghtml).replace("{{categories}}",str(categories)) + imggrid=headertemplate + imggrid+=imagegridtemplate.replace("{{imagecontainers}}",imghtml).replace("{{categories}}",str(categories).replace("{","").replace("}","")) + imggrid+=footertemplate + return imggrid @staticmethod def generateIIIFCollections(outpath, deploypath, imagespaths, prefixnamespace): + apihtml = "SwaggerUI
    " + apijson = {"openapi": "3.0.1", "info": {"title": str(deploypath) + " IIIF", "description": "IIIF API of " + str(deploypath)},"servers": [{"url": str(deploypath)}], "paths": {}} + apijson["paths"]["/iiif/collection/"] = {"get": {"tags": ["IIIF"], + "summary": "Retrieves IIIF Collections of "+str(deploypath), + "description": "Retrieves the IIIF Collections of this IIIF API", + "operationId": "iiif-collections", + "parameters": [], + "responses": { + "200": { + "description": "Success", + "content": { + "text/plain": { + "example": None}}, + "application/json": { + "schema": { + "example": None}, + "example": None}, + "text/json": { + "schema": { + "example": None}, + "example": None}}}}} + apijson["paths"]["/iiif/{id}/manifest"] = {"get": {"tags": ["IIIF"], + "summary": "Retrieves a IIIF manifest with a given ID", + "description": "Retrieves a IIIF manifest with a given ID", + "operationId": "iiif-manifest", + "parameters": [{"in":"path","name":"id","required":True,"schema": {"type": "string"}}], + "responses": { + "200": { + "description": "Success", + "content": { + "text/plain": { + "example": None}}, + "application/json": { + "schema": { + "example": None}, + "example": None}, + "text/json": { + "schema": { + "example": None}, + "example": None}}}}} if not os.path.exists(outpath + "/iiif/collection/"): os.makedirs(outpath + "/iiif/collection/") if os.path.exists(outpath + "/iiif/collection/iiifcoll.json"): @@ -184,6 +234,8 @@ def generateIIIFCollections(outpath, deploypath, imagespaths, prefixnamespace): seenurls = set() for imgpath in sorted(imagespaths, key=lambda k: k['label'], reverse=False): curclass = "main" + if "main" not in collections: + collections["main"]={"items":[]} if "class" in imgpath and imgpath["class"] != "": curclass = imgpath["class"] if curclass not in collections: @@ -215,4 +267,10 @@ def generateIIIFCollections(outpath, deploypath, imagespaths, prefixnamespace): iiifindex = """
    """ f = open(outpath + "/iiif/index.html", "w", encoding="utf-8") f.write(iiifindex) + f.close() + f = open(outpath + "/iiif/api.html", "w", encoding="utf-8") + f.write(apihtml) + f.close() + f = open(outpath + "/iiif/api.json", "w", encoding="utf-8") + f.write(json.dumps(apijson)) f.close() \ No newline at end of file diff --git a/util/export/api/solidexporter.py b/util/export/api/solidexporter.py index 0032d055b..a7669b0f2 100644 --- a/util/export/api/solidexporter.py +++ b/util/export/api/solidexporter.py @@ -4,7 +4,7 @@ from rdflib import Literal from rdflib import Graph -from doc.docutils import DocUtils +from ...doc.docutils import DocUtils class SolidExporter: diff --git a/util/export/data/exporter/layer/__init__.py b/util/export/data/exporter/layer/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/util/export/data/exporter/rdf/__init__.py b/util/export/data/exporter/rdf/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/util/export/data/exporter/rdf/voidexporter.py b/util/export/data/exporter/rdf/voidexporter.py deleted file mode 100644 index 47747ee83..000000000 --- a/util/export/data/exporter/rdf/voidexporter.py +++ /dev/null @@ -1,140 +0,0 @@ -from rdflib import URIRef, Literal, Graph -from .....doc.docconfig import DocConfig - -from .....doc.docutils import DocUtils - - -class VoidExporter: - - @staticmethod - def createVoidDataset(dsname,prefixnamespace,prefixshort,repository,deploypath,outpath,licenseuri,modtime,language,stats,subjectstorender,prefixes,classtree=None,propstats=None,nonnscount=None,nscount=None,objectmap=None,startconcept=None): - g=Graph() - g.bind("voaf","http://purl.org/vocommons/voaf#") - g.bind("vext", "http://ldf.fi/void-ext#") - g.bind("vann", "http://purl.org/vocab/vann/") - if dsname==None or dsname=="": - dsname="dataset" - voidds=prefixnamespace+dsname - if repository!=None and repository!="" and repository.startswith("http"): - g.add((URIRef(repository), URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), - URIRef("http://www.w3.org/ns/adms#AssetRepository"))) - g.add((URIRef(repository), URIRef("http://www.w3.org/ns/dcat#accessURL"), - Literal(str(repository),datatype="http://www.w3.org/2001/XMLSchema#anyURI"))) - g.add((URIRef(repository), URIRef("http://www.w3.org/ns/dcat#dataset"), - URIRef(voidds))) - g.add((URIRef(repository), URIRef("http://www.w3.org/2000/01/rdf-schema#label"), - Literal("Repository for "+str(dsname), lang="en"))) - g.add((URIRef(voidds),URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"),URIRef("http://rdfs.org/ns/void#Dataset"))) - g.add((URIRef(voidds), URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), URIRef("http://www.w3.org/ns/adms#Asset"))) - g.add((URIRef(voidds), URIRef("http://www.w3.org/2000/01/rdf-schema#label"), - Literal(dsname,lang="en"))) - g.add((URIRef(voidds), URIRef("http://purl.org/dc/terms/title"), - Literal(dsname,lang="en"))) - if language!=None and language!="": - g.add((URIRef(voidds), URIRef("http://purl.org/dc/elements/1.1/language"), - URIRef("http://www.lexvo.org/page/iso639-1/"+str(language)))) - g.add((URIRef(voidds), URIRef("http://purl.org/dc/terms/modified"), - Literal(modtime,datatype="http://www.w3.org/2001/XMLSchema#dateTime"))) - if licenseuri!=None: - g.add((URIRef(voidds), URIRef("http://purl.org/dc/terms/license"), - URIRef(licenseuri))) - g.add((URIRef(voidds), URIRef("http://rdfs.org/ns/void#dataDump"), - URIRef(deploypath+"/index.ttl"))) - g.add((URIRef(voidds), URIRef("http://xmlns.com/foaf/0.1/homepage"), - URIRef(deploypath))) - g.add((URIRef(voidds), URIRef("http://xmlns.com/foaf/0.1/page"), - URIRef(deploypath+"/index.html"))) - g.add((URIRef(voidds), URIRef("http://rdfs.org/ns/void#dataDump"), - URIRef(deploypath+"/index.ttl"))) - g.add((URIRef(voidds), URIRef("http://www.w3.org/ns/dcat#distribution"), - URIRef(voidds+"_dist_ttl"))) - g.add((URIRef(voidds + "_dist_ttl"), URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), URIRef("http://www.w3.org/ns/adms#AssetDistribution"))) - g.add((URIRef(voidds+"_dist_ttl"), URIRef("http://www.w3.org/2000/01/rdf-schema#label"), - Literal(dsname+" TTL Distribution",lang="en"))) - g.add((URIRef(voidds+"_dist_ttl"), URIRef("http://www.w3.org/ns/dcat#downloadURL"), - Literal(deploypath+"/index.ttl",datatype="http://www.w3.org/2001/XMLSchema#anyURI"))) - g.add((URIRef(voidds+"_dist_ttl"), URIRef("http://www.w3.org/ns/dcat#mediaType"), - URIRef("http://www.w3.org/ns/formats/Turtle"))) - g.add((URIRef(voidds), URIRef("http://rdfs.org/ns/void#feature"), - URIRef("http://www.w3.org/ns/formats/Turtle"))) - g.add((URIRef(voidds), URIRef("http://rdfs.org/ns/void#feature"), - URIRef("http://www.w3.org/ns/formats/RDFa"))) - if startconcept!=None and startconcept!="": - g.add((URIRef(voidds), URIRef("http://rdfs.org/ns/void#rootResource"), URIRef(startconcept.replace("index.html","")))) - g.add((URIRef(voidds), URIRef("http://rdfs.org/ns/void#exampleResource"), URIRef(startconcept.replace("index.html","")))) - for stat in stats: - g.add((URIRef(voidds), URIRef(stat),Literal(stats[stat], datatype="http://www.w3.org/2001/XMLSchema#integer"))) - g.add((URIRef(voidds), URIRef("http://rdfs.org/ns/void#uriSpace"), - Literal(prefixnamespace,datatype="http://www.w3.org/2001/XMLSchema#string"))) - g.add((URIRef(voidds), URIRef("http://purl.org/vocab/vann/preferredNamespaceUri"), - Literal(prefixnamespace, datatype="http://www.w3.org/2001/XMLSchema#anyURI"))) - g.add((URIRef(voidds), URIRef("http://purl.org/vocab/vann/preferredNamespacePrefix"), - Literal(prefixshort, datatype="http://www.w3.org/2001/XMLSchema#string"))) - for ns_prefix, namespace in g.namespaces(): - g.add((URIRef(voidds), URIRef("http://rdfs.org/ns/void#vocabulary"),URIRef(namespace))) - g.add((URIRef(namespace), URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), URIRef("http://purl.org/vocommons/voaf#Vocabulary"))) - if "nstolabel" in prefixes and str(namespace) in prefixes["nstolabel"]: - g.add((URIRef(namespace), URIRef("http://www.w3.org/2000/01/rdf-schema#label"), - Literal(prefixes["nstolabel"][str(namespace)],lang="en"))) - else: - g.add((URIRef(namespace), URIRef("http://www.w3.org/2000/01/rdf-schema#label"), - Literal(str(ns_prefix)+" Vocabulary",lang="en"))) - g.add((URIRef(namespace), URIRef("http://purl.org/vocab/vann/preferredNamespaceUri"), - Literal(namespace,datatype="http://www.w3.org/2001/XMLSchema#anyURI"))) - g.add((URIRef(namespace), URIRef("http://purl.org/vocab/vann/preferredNamespacePrefix"), - Literal(ns_prefix,datatype="http://www.w3.org/2001/XMLSchema#string"))) - g.add((URIRef(namespace+"_"+str(dsname)+"_occ"), URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), - URIRef("http://purl.org/vocommons/voaf#DatasetOccurrence"))) - g.add((URIRef(namespace+"_"+str(dsname)+"_occ"), URIRef("http://www.w3.org/2000/01/rdf-schema#label"), - Literal("Occurrences of vocabulary "+str(namespace)+" in "+dsname))) - if nscount!=None and str(namespace) in nscount: - g.add((URIRef(prefixnamespace+str(ns_prefix)+"_"+str(dsname)+"_occ"), URIRef("http://purl.org/vocommons/voaf#occurrences"), - Literal(str(nscount[str(namespace)]),datatype="http://www.w3.org/2001/XMLSchema#integer"))) - g.add((URIRef(namespace), URIRef("http://purl.org/vocommons/voaf#usageInDataset"), URIRef(namespace+"_"+str(dsname)+"_occ"))) - g.add((URIRef(namespace+"_"+str(dsname)+"_occ"), URIRef("http://purl.org/vocommons/voaf#inDataset"), URIRef(voidds))) - if str(namespace) in DocConfig.namespaceToTopic: - for entry in DocConfig.namespaceToTopic[str(namespace)]: - g.add((URIRef(voidds), URIRef("http://www.w3.org/ns/dcat#keyword"), - Literal(DocUtils.shortenURI(entry["uri"]).replace("_", " "), lang="en"))) - g.add((URIRef(voidds), URIRef("http://purl.org/dc/terms/subject"),URIRef(entry["uri"]))) - g.add((URIRef(entry["uri"]),URIRef("http://www.w3.org/2000/01/rdf-schema#label"),Literal(entry["label"],lang="en"))) - for pred in propstats: - cururi=voidds+"_"+DocUtils.shortenURI(pred) - g.add((URIRef(voidds),URIRef("http://rdfs.org/ns/void#propertyPartition"),URIRef(cururi))) - g.add((URIRef(cururi), URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"),URIRef("http://rdfs.org/ns/void#Dataset"))) - g.add((URIRef(cururi), URIRef("http://www.w3.org/2000/01/rdf-schema#label"),Literal("Property Partition: " + str(DocUtils.shortenURI(pred)), lang="en"))) - g.add((URIRef(cururi),URIRef("http://rdfs.org/ns/void#property"),URIRef(pred))) - g.add((URIRef(cururi),URIRef("http://rdfs.org/ns/void#triples"),Literal(str(propstats[pred]["triples"]),datatype="http://www.w3.org/2001/XMLSchema#integer"))) - subjectstorender.add(URIRef(cururi)) - for item in classtree["core"]["data"]: - if item["type"]=="class": - cururi = voidds +"_"+ DocUtils.shortenURI(item["id"]) - g.add((URIRef(voidds), URIRef("http://rdfs.org/ns/void#classPartition"), URIRef(cururi))) - g.add((URIRef(cururi), URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"),URIRef("http://rdfs.org/ns/void#Dataset"))) - g.add((URIRef(cururi), URIRef("http://www.w3.org/2000/01/rdf-schema#label"),Literal("Class Partition: " + str(DocUtils.shortenURI(item["id"])),lang="en"))) - g.add((URIRef(cururi), URIRef("http://rdfs.org/ns/void#class"), URIRef(item["id"]))) - if item["id"] in objectmap: - - g.add((URIRef(cururi), URIRef("http://rdfs.org/ns/void#entities"),Literal(str(objectmap[item["id"]]), datatype="http://www.w3.org/2001/XMLSchema#integer"))) - #subjectstorender.add(URIRef(cururi)) - for prop in nonnscount: - for ns in nonnscount[prop]: - cururi=voidds+"_"+ns.replace("http://","").replace("https://","").replace("/","_").replace("#","_") - g.add((URIRef(cururi), URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"),URIRef("http://rdfs.org/ns/void#Linkset"))) - g.add((URIRef(cururi), URIRef("http://www.w3.org/2000/01/rdf-schema#label"),Literal("Linkset: "+str(DocUtils.shortenURI(voidds))+" - "+str(DocUtils.getLabelForObject(URIRef(ns),g,prefixes)),lang="en"))) - g.add((URIRef(cururi), URIRef("http://rdfs.org/ns/void#subjectsTarget"),URIRef(voidds))) - g.add((URIRef(cururi), URIRef("http://rdfs.org/ns/void#objectsTarget"),URIRef(ns))) - g.add((URIRef(cururi), URIRef("http://rdfs.org/ns/void#linkPredicate"),URIRef(prop))) - g.add((URIRef(cururi), URIRef("http://rdfs.org/ns/void#triples"),Literal(str(nonnscount[prop][ns]),datatype="http://www.w3.org/2001/XMLSchema#integer"))) - subjectstorender.add(URIRef(cururi)) - g.serialize(outpath+"/void.ttl", encoding="utf-8") - return {"graph":g,"subjects":subjectstorender} - - @staticmethod - def toHTML(stats,deploypath): - result="
    Dataset Statistics [VOID]" - result+="" - for stat in stats: - result+="" - result+="
    PropertyValue
    "+str(DocUtils.shortenURI(stat))+""+str(stats[stat])+" (xsd:integer)
    " - return result \ No newline at end of file diff --git a/util/export/data/exporter/rdf/geoexporter.py b/util/export/data/geoexporter.py similarity index 99% rename from util/export/data/exporter/rdf/geoexporter.py rename to util/export/data/geoexporter.py index eebeea90d..333af87bb 100644 --- a/util/export/data/exporter/rdf/geoexporter.py +++ b/util/export/data/geoexporter.py @@ -1,7 +1,7 @@ import json import os -from ......util.doc.docutils import DocUtils +from ...doc.docutils import DocUtils class GeoExporter: diff --git a/util/export/data/exporter/rdf/graphexporter.py b/util/export/data/graphexporter.py similarity index 99% rename from util/export/data/exporter/rdf/graphexporter.py rename to util/export/data/graphexporter.py index d94b623c7..81a0ecd3a 100644 --- a/util/export/data/exporter/rdf/graphexporter.py +++ b/util/export/data/graphexporter.py @@ -2,7 +2,7 @@ import json import random -from .....doc.docutils import DocUtils +from ...doc.docutils import DocUtils class GraphExporter: diff --git a/util/export/data/htmlexporter.py b/util/export/data/htmlexporter.py new file mode 100644 index 000000000..82ccadbec --- /dev/null +++ b/util/export/data/htmlexporter.py @@ -0,0 +1,900 @@ +from ...doc.docutils import DocUtils +from ...doc.literalutils import LiteralUtils +from ...doc.docutils import DocConfig +from ...sparqlutils import SPARQLUtils +from ..pages.bibpage import BibPage +from ..pages.owltimepage import OWLTimePage +from rdflib import URIRef, Graph, BNode, Literal +import re +import os +import json +import urllib +import traceback + +from ..api.iiifexporter import IIIFAPIExporter +from ..pages.geometryviewpage import GeometryViewPage +from ..pages.lexiconpage import LexiconPage +from ..pages.observationpage import ObservationPage +from ..pages.personpage import PersonPage + + +class HTMLExporter(): + listthreshold = 5 + maxlistthreshold = 1500 + + featurecollectionspaths = {} + iiifmanifestpaths = {"default": []} + imagetoURI = {} + geocache = {} + + def __init__(self, prefixes, prefixnamespace, prefixnsshort, license, labellang, outpath, + metadatatable, generatePagesForNonNS, apis,templates,namespaceshort,typeproperty="http://www.w3.org/1999/02/22-rdf-syntax-ns#type", + imagemetadata=None,localOptimized=False, + deploypath="", logoname="", offlinecompat=False): + self.prefixes = prefixes + self.prefixnamespace = prefixnamespace + self.outpath = outpath + self.deploypath = deploypath + self.metadatatable = metadatatable + self.logoname = logoname + self.templates = templates + self.localOptimized=localOptimized + self.apis = apis + self.has3d=False + self.labellang = labellang + self.license = license + self.licenseuri="" + self.publisher="" + self.publishingorg="" + self.datasettitle="" + self.typeproperty=typeproperty + self.prefixnsshort = prefixnsshort + self.imagemetadata = imagemetadata + self.namespaceshort = namespaceshort + self.generatePagesForNonNS = generatePagesForNonNS + self.offlinecompat = offlinecompat + + def replaceStandardVariables(self, template, subject, checkdepth, indexpage): + template = template.replace("{{indexpage}}", str(indexpage)).replace("{{subject}}", str(subject)).replace( + "{{relativedepth}}", str(checkdepth)) \ + .replace("{{versionurl}}", DocConfig.versionurl).replace("{{version}}", DocConfig.version).replace( + "{{deploypath}}", self.deploypath) \ + .replace("{{publishingorg}}", self.publishingorg).replace("{{publisher}}", self.publisher).replace( + "{{datasettitle}}", self.datasettitle) \ + .replace("{{logo}}", self.logoname) + return template + + def createHTML(self, savepath, predobjs, subject, baseurl, subpreds, graph, searchfilename, classtreename, + uritotreeitem, curlicense, subjectstorender, postprocessing, nonnsmap=None, nonns=False, + foundlabel=""): + tablecontents = "" + metadatatablecontents = "" + geojsonrep = None + epsgcode = "" + foundmedia = {"audio": {}, "video": {}, "image": {}, "mesh": {}} + savepath = savepath.replace("\\", "/") + checkdepth = 0 + if not nonns: + checkdepth = DocUtils.checkDepthFromPath(savepath, baseurl, subject) + logo = "" + if self.logoname is not None and self.logoname != "": + logo = "\"logo\"  " + textannos = [] + foundvals = set() + imageannos = [] + annobodies = [] + image3dannos = [] + predobjmap = {} + curtypes = set() + comment = {} + parentclass = None + inverse = False + dateprops = [] + timeobj = None + tablecontentcounter = -1 + metadatatablecontentcounter = -1 + if uritotreeitem is not None and str(subject) in uritotreeitem and uritotreeitem[str(subject)][-1][ + "parent"].startswith("http"): + parentclass = str(uritotreeitem[str(subject)][-1]["parent"]) + if parentclass not in uritotreeitem: + uritotreeitem[parentclass] = [ + {"id": parentclass, "parent": "#", "type": "class", "text": DocUtils.shortenURI(str(parentclass)), + "data": {}}] + # print(uritotreeitem[parentclass]) + uritotreeitem[parentclass][-1]["instancecount"] = 0 + ttlf = Graph(bind_namespaces="rdflib") + # ttlf = open(savepath + "/index.ttl", "w", encoding="utf-8") + if parentclass is not None: + uritotreeitem[parentclass][-1]["data"]["to"] = {} + uritotreeitem[parentclass][-1]["data"]["from"] = {} + hasnonns = set() + thetypes = set() + itembibtex = "" + collections = set() + if predobjs is not None: + for tup in sorted(predobjs, key=lambda tup: tup[0]): + if str(tup[0]) not in predobjmap: + predobjmap[str(tup[0])] = [] + predobjmap[str(tup[0])].append(tup[1]) + if parentclass is not None and str(tup[0]) not in uritotreeitem[parentclass][-1]["data"]["to"]: + uritotreeitem[parentclass][-1]["data"]["to"][str(tup[0])] = {} + uritotreeitem[parentclass][-1]["data"]["to"][str(tup[0])]["instancecount"] = 0 + if parentclass is not None: + uritotreeitem[parentclass][-1]["data"]["to"][str(tup[0])]["instancecount"] += 1 + uritotreeitem[parentclass][-1]["instancecount"] += 1 + if isinstance(tup[1], URIRef): + for item in graph.objects(tup[1], URIRef(self.typeproperty)): + thetypes.add(str(item)) + if parentclass is not None: + if item not in uritotreeitem[parentclass][-1]["data"]["to"][str(tup[0])]: + uritotreeitem[parentclass][-1]["data"]["to"][str(tup[0])][item] = 0 + uritotreeitem[parentclass][-1]["data"]["to"][str(tup[0])][item] += 1 + if baseurl not in str(tup[1]) and str(tup[0]) != self.typeproperty: + hasnonns.add(str(tup[1])) + if nonnsmap is not None: + if str(tup[1]) not in nonnsmap: + nonnsmap[str(tup[1])] = set() + nonnsmap[str(tup[1])].add(subject) + for tup in sorted(predobjmap): + if self.metadatatable and tup not in SPARQLUtils.labelproperties and DocUtils.shortenURI(str(tup), + True) in SPARQLUtils.metadatanamespaces: + thetable = metadatatablecontents + metadatatablecontentcounter += 1 + if metadatatablecontentcounter % 2 == 0: + thetable += "" + else: + thetable += "" + else: + thetable = tablecontents + tablecontentcounter += 1 + if tablecontentcounter % 2 == 0: + thetable += "" + else: + thetable += "" + if str(tup) == self.typeproperty: + for tp in predobjmap[tup]: + thetypes.add(str(tp)) + curtypes.add(str(tp)) + if str(tp) in SPARQLUtils.collectionclasses: + uritotreeitem[str(tp)][-1]["instancecount"] += 1 + collections.add(SPARQLUtils.collectionclasses[str(tp)]) + if str(tp) in DocConfig.bibtextypemappings: + itembibtex = "
    [BIBTEX]
    " + str(
+                                BibPage.resolveBibtexReference(graph.predicate_objects(subject), subject,
+                                                               graph)) + "
    " + thetable = HTMLExporter.formatPredicate(tup, baseurl, checkdepth, thetable, graph, inverse, + self.labellang, self.prefixes) + if str(tup) in SPARQLUtils.labelproperties: + for lab in predobjmap[tup]: + if lab.language == self.labellang: + foundlabel = lab + if foundlabel == "": + foundlabel = str(predobjmap[tup][0]) + if str(tup) in SPARQLUtils.commentproperties: + comment[str(tup)] = str(predobjmap[tup][0]) + if len(predobjmap[tup]) > 0: + thetable += "" + if len(predobjmap[tup]) > HTMLExporter.listthreshold: + thetable += "
    " + str(len(predobjmap[tup])) + " values" + if len(predobjmap[tup]) > 1: + thetable += "
      " + labelmap = {} + itemcounter = 0 + for item in predobjmap[tup]: + if itemcounter >= HTMLExporter.maxlistthreshold: + break + if ("POINT" in str(item).upper() or "POLYGON" in str(item).upper() or "LINESTRING" in str( + item).upper()) and tup in SPARQLUtils.valueproperties and self.typeproperty in predobjmap and URIRef( + "http://www.w3.org/ns/oa#WKTSelector") in predobjmap[self.typeproperty]: + image3dannos.append({"value": str(item)}) + elif " 1: + labelmap[res["label"]] += "
    • " + str(res["html"]) + "
      • " + else: + labelmap[res["label"]] += str(res["html"]) + itemcounter += 1 + for lab in sorted(labelmap): + thetable += str(labelmap[lab]) + if len(predobjmap[tup]) >= HTMLExporter.maxlistthreshold: + tablecontents += "
    • (...)
      • " + if len(predobjmap[tup]) > 1: + thetable += "
    " + if len(predobjmap[tup]) > HTMLExporter.listthreshold: + thetable += "
    " + thetable += "" + else: + thetable += "" + thetable += "" + if self.metadatatable and tup not in SPARQLUtils.labelproperties and DocUtils.shortenURI(str(tup), + True) in SPARQLUtils.metadatanamespaces: + metadatatablecontents = thetable + else: + tablecontents = thetable + subpredsmap = {} + if subpreds != None: + for tup in sorted(subpreds, key=lambda tup: tup[1]): + if str(tup[1]) not in subpredsmap: + subpredsmap[str(tup[1])] = [] + subpredsmap[str(tup[1])].append(tup[0]) + if parentclass != None and str(tup[1]) not in uritotreeitem[parentclass][-1]["data"]["from"]: + uritotreeitem[parentclass][-1]["data"]["from"][str(tup[1])] = {} + uritotreeitem[parentclass][-1]["data"]["from"][str(tup[1])]["instancecount"] = 0 + if isinstance(tup[0], URIRef): + for item in graph.objects(tup[0], URIRef(self.typeproperty)): + if parentclass != None: + if item not in uritotreeitem[parentclass][-1]["data"]["from"][str(tup[1])]: + uritotreeitem[parentclass][-1]["data"]["from"][str(tup[1])][item] = 0 + uritotreeitem[parentclass][-1]["data"]["from"][str(tup[1])][item] += 1 + for tup in subpredsmap: + tablecontentcounter += 1 + if tablecontentcounter % 2 == 0: + tablecontents += "" + else: + tablecontents += "" + tablecontents = HTMLExporter.formatPredicate(tup, baseurl, checkdepth, tablecontents, graph, True, + self.labellang, self.prefixes) + if len(subpredsmap[tup]) > 0: + tablecontents += "" + if len(subpredsmap[tup]) > HTMLExporter.listthreshold: + tablecontents += "
    " + str(len(subpredsmap[tup])) + " values" + if len(subpredsmap[tup]) > 1: + tablecontents += "
      " + labelmap = {} + itemcounter = 0 + for item in subpredsmap[tup]: + if itemcounter >= HTMLExporter.maxlistthreshold: + break + if subjectstorender is not None and item not in subjectstorender and baseurl in str(item): + postprocessing.add((item, URIRef(tup), subject)) + res = HTMLExporter.createHTMLTableValueEntry(subject, tup, item, None, graph, + baseurl, checkdepth, geojsonrep, foundmedia, + imageannos, + textannos, image3dannos, annobodies, None, True, + nonns, self.labellang, self.typeproperty, + self.namespaceshort, self.generatePagesForNonNS, + self.prefixes) + foundmedia = res["foundmedia"] + imageannos = res["imageannos"] + image3dannos = res["image3dannos"] + annobodies = res["annobodies"] + # print("POSTPROC ANNO BODIES "+str(annobodies)) + if nonns and str(tup) != self.typeproperty: + hasnonns.add(str(item)) + if nonns: + geojsonrep = res["geojson"] + if res["label"] not in labelmap: + labelmap[res["label"]] = "" + if len(subpredsmap[tup]) > 1: + labelmap[res["label"]] += "
    • " + str(res["html"]) + "
      • " + else: + labelmap[res["label"]] += str(res["html"]) + itemcounter += 1 + for lab in sorted(labelmap): + tablecontents += str(labelmap[lab]) + if len(subpredsmap[tup]) >= HTMLExporter.maxlistthreshold: + tablecontents += "
    • (...)
      • " + if len(subpredsmap[tup]) > 1: + tablecontents += "
    " + if len(subpredsmap[tup]) > HTMLExporter.listthreshold: + tablecontents += "
    " + tablecontents += "" + else: + tablecontents += "" + tablecontents += "" + if self.licenseuri != None: + ttlf.add((subject, URIRef("http://purl.org/dc/elements/1.1/license"), URIRef(self.licenseuri))) + if self.apis["solidexport"] != None: + ttlf.add((subject, URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + URIRef("http://www.w3.org/ns/ldp#Resource"))) + ttlf.add((subject, URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + URIRef("https://www.iana.org/assignments/media-types/text/turtle#Resource"))) + nonnslink = "" + if nonns: + completesavepath = savepath + nonnslink = "
    This page describes linked instances to the concept " + str(foundlabel) + " (" + str(DocUtils.shortenURI( + subject)) + ") in this knowledge graph. It is defined here
    " + else: + completesavepath = savepath + "/index.html" + if not nonns: + if os.path.exists(savepath): + try: + ttlf.serialize(savepath + "/index.ttl", encoding="utf-8") + with open(savepath + "/index.json", 'w', encoding='utf-8') as f: + f.write(json.dumps(predobjmap)) + f.close() + except Exception as e: + print(e) + print(traceback.format_exc()) + with open(completesavepath, 'w', encoding='utf-8') as f: + searchfilelink = DocUtils.generateRelativeLinkFromGivenDepth(baseurl, checkdepth, searchfilename, False) + classtreelink = DocUtils.generateRelativeLinkFromGivenDepth(baseurl, checkdepth, classtreename, False) + csslink = DocUtils.generateRelativeLinkFromGivenDepth(baseurl, checkdepth, "style.css", False) + startscriptlink = DocUtils.generateRelativeLinkFromGivenDepth(baseurl, checkdepth, "startscripts.js", + False) + proprelationslink = DocUtils.generateRelativeLinkFromGivenDepth(baseurl, checkdepth, "proprelations.js", + False) + epsgdefslink = DocUtils.generateRelativeLinkFromGivenDepth(baseurl, checkdepth, "epsgdefs.js", False) + vowlresultlink = DocUtils.generateRelativeLinkFromGivenDepth(baseurl, checkdepth, "vowl_result.js", + False) + if geojsonrep != None: + myexports = self.templates["geoexports"] + else: + myexports = self.templates["nongeoexports"] + relpath = DocUtils.generateRelativePathFromGivenDepth(checkdepth) + if foundlabel == None or foundlabel == "": + foundlabel = DocUtils.shortenURI(str(subject)) + f.write(self.replaceStandardVariables(self.templates["htmltemplate"], subject, checkdepth, "false").replace( + "{{iconprefixx}}", (relpath + "icons/" if self.offlinecompat else "")).replace("{{baseurl}}", + baseurl).replace( + "{{relativepath}}", DocUtils.generateRelativePathFromGivenDepth(checkdepth)).replace( + "{{relativedepth}}", str(checkdepth)).replace("{{prefixpath}}", self.prefixnamespace).replace( + "{{toptitle}}", foundlabel).replace( + "{{startscriptpath}}", startscriptlink).replace("{{epsgdefspath}}", epsgdefslink).replace( + "{{bibtex}}", itembibtex).replace("{{vowlpath}}", vowlresultlink).replace("{{proprelationpath}}", + proprelationslink).replace( + "{{stylepath}}", csslink).replace("{{title}}", + "" + str( + foundlabel) + "").replace( + "{{baseurl}}", baseurl).replace("{{tablecontent}}", tablecontents).replace("{{description}}", + "").replace( + "{{scriptfolderpath}}", searchfilelink).replace("{{classtreefolderpath}}", classtreelink).replace( + "{{exports}}", myexports).replace("{{nonnslink}}", str(nonnslink)).replace("{{subjectencoded}}", + urllib.parse.quote( + str(subject)))) + for comm in comment: + f.write(self.templates["htmlcommenttemplate"].replace("{{comment}}", + DocUtils.shortenURI(comm) + ":" + comment[comm])) + # for fval in foundvals: + # f.write(templates["htmlcommenttemplate"].replace("{{comment}}", "Value "+ DocUtils.shortenURI(str(fval[0]))+": " + str(fval[1]) + "")) + if len(foundmedia["mesh"]) > 0 and len(image3dannos) > 0: + if self.apis["iiif"]: + self.iiifmanifestpaths["default"].append( + IIIFAPIExporter.generateIIIFManifest(graph, self.outpath, self.deploypath, + foundmedia["mesh"], image3dannos, annobodies, + str(subject), self.prefixnamespace, self.imagetoURI, + self.imagemetadata, DocConfig.metadatanamespaces, + foundlabel, comment, thetypes, predobjmap, "Model")) + for anno in image3dannos: + if ("POINT" in anno["value"].upper() or "POLYGON" in anno["value"].upper() or "LINESTRING" in + anno["value"].upper()): + f.write(self.templates["threejstemplate"].replace("{{wktstring}}", anno["value"]).replace( + "{{meshurls}}", str(list(foundmedia["mesh"]))).replace("{{relativepath}}", + DocUtils.generateRelativePathFromGivenDepth( + checkdepth))) + elif len(foundmedia["mesh"]) > 0 and len(image3dannos) == 0: + print("Found 3D Model: " + str(foundmedia["mesh"])) + if self.apis["iiif"]: + self.iiifmanifestpaths["default"].append( + IIIFAPIExporter.generateIIIFManifest(graph, self.outpath, self.deploypath, + foundmedia["mesh"], image3dannos, annobodies, + str(subject), self.prefixnamespace, self.imagetoURI, + self.imagemetadata, DocConfig.metadatanamespaces, + foundlabel, comment, thetypes, predobjmap, "Model")) + for curitem in foundmedia["mesh"]: + format = "ply" + if ".nxs" in curitem or ".nxz" in curitem: + format = "nexus" + self.has3d = True + elif format == "gltf": + f.write(self.templates["threejstemplate"].replace("{{wktstring}}", "").replace("{{meshurls}}", + str(list( + foundmedia[ + "mesh"]))).replace( + "{{relativepath}}", DocUtils.generateRelativePathFromGivenDepth(checkdepth))) + f.write(self.templates["threejstemplate"].replace("{{wktstring}}", "").replace("{{meshurls}}", + str(list(foundmedia[ + "mesh"]))).replace( + "{{relativepath}}", DocUtils.generateRelativePathFromGivenDepth(checkdepth))) + # f.write(templates["3dtemplate"].replace("{{meshurl}}",curitem).replace("{{meshformat}}",format)) + break + elif len(foundmedia["mesh"]) == 0 and len(image3dannos) > 0: + for anno in image3dannos: + if ("POINT" in anno["value"].upper() or "POLYGON" in anno["value"].upper() or "LINESTRING" in + anno["value"].upper()): + f.write(self.templates["threejstemplate"].replace("{{wktstring}}", anno["value"]).replace( + "{{meshurls}}", "[]").replace("{{relativepath}}", + DocUtils.generateRelativePathFromGivenDepth(checkdepth))) + carousel = "image" + if len(foundmedia["image"]) > 3: + carousel = "carousel-item active" + f.write(self.templates["imagecarouselheader"]) + # if self.apis["iiif"] and len(annobodies)>0: + # if target not in imagetoURI: + # imagetoURI[target]={"uri":{str(subject):{"bodies":[]}}} + # if str(subject) not in imagetoURI[target]: + # imagetoURI[target]["uri"][str(subject)]={"bodies":[]} + # if str(subject) not in imagetoURI[target]: + # imagetoURI[target]["uri"][str(subject)]["bodies"]+=annobodies + if len(imageannos) > 0 and len(foundmedia["image"]) > 0: + if self.apis["iiif"]: + self.iiifmanifestpaths["default"].append( + IIIFAPIExporter.generateIIIFManifest(graph, self.outpath, self.deploypath, + foundmedia["image"], imageannos, annobodies, + str(subject), self.prefixnamespace, self.imagetoURI, + self.imagemetadata, DocConfig.metadatanamespaces, + foundlabel, comment, thetypes, predobjmap, "Image")) + for image in foundmedia["image"]: + if image not in self.imagetoURI or "uri" not in self.imagetoURI[image]: + self.imagetoURI[image] = {"uri": {}} + if not str(subject) in self.imagetoURI[image]["uri"]: + self.imagetoURI[image]["uri"][str(subject)] = {"bodies": []} + annostring = "" + for anno in imageannos: + annostring += anno["value"].replace("", + "") + f.write(self.templates["imageswithannotemplate"].replace("{{carousel}}", + carousel + "\" style=\"position: relative;display: inline-block;").replace( + "{{image}}", str(image)).replace("{{svganno}}", annostring).replace("{{imagetitle}}", + str(image)[ + 0:str(image).rfind( + '.')])) + if len(foundmedia["image"]) > 3: + carousel = "carousel-item" + elif len(foundmedia["image"]) > 0: + if self.apis["iiif"]: + self.iiifmanifestpaths["default"].append( + IIIFAPIExporter.generateIIIFManifest(graph, self.outpath, self.deploypath, + foundmedia["image"], imageannos, annobodies, + str(subject), self.prefixnamespace, self.imagetoURI, + self.imagemetadata, DocConfig.metadatanamespaces, + foundlabel, comment, thetypes, predobjmap, "Image")) + for image in foundmedia["image"]: + if image not in self.imagetoURI or "uri" not in self.imagetoURI[image]: + self.imagetoURI[image] = {"uri": {}} + if not str(subject) in self.imagetoURI[image]["uri"]: + self.imagetoURI[image]["uri"][str(subject)] = {"bodies": []} + if image == "" in image: + f.write(self.templates["imagestemplatesvg"].replace("{{carousel}}", carousel).replace( + "{{image}}", str(image.replace("", "")))) + else: + f.write(self.templates["imagestemplatesvg"].replace("{{carousel}}", carousel).replace( + "{{image}}", str(image))) + else: + f.write(self.templates["imagestemplate"].replace("{{carousel}}", carousel).replace("{{image}}", + str(image)).replace( + "{{imagetitle}}", str(image)[0:str(image).rfind('.')])) + if len(foundmedia["image"]) > 3: + carousel = "carousel-item" + if len(foundmedia["image"]) > 3: + f.write(self.templates["imagecarouselfooter"]) + if len(textannos) > 0: + for textanno in textannos: + if isinstance(textanno, dict): + if "src" in textanno: + f.write("" + str( + textanno["exact"]) + "") + else: + f.write("" + str(textanno["exact"]) + "") + if len(foundmedia["audio"]) > 0 and self.apis["iiif"]: + self.iiifmanifestpaths["default"].append( + IIIFAPIExporter.generateIIIFManifest(graph, self.outpath, self.deploypath, foundmedia["audio"], + None, None, str(subject), self.prefixnamespace, + self.imagetoURI, + self.imagemetadata, DocConfig.metadatanamespaces, + foundlabel, comment, thetypes, predobjmap, "Audio")) + for audio in foundmedia["audio"]: + self.imagetoURI[audio] = {"uri": str(subject)} + f.write(self.templates["audiotemplate"].replace("{{audio}}", str(audio))) + if len(foundmedia["video"]) > 0 and self.apis["iiif"]: + self.iiifmanifestpaths["default"].append( + IIIFAPIExporter.generateIIIFManifest(graph, self.outpath, self.deploypath, foundmedia["video"], + None, None, str(subject), self.prefixnamespace, + self.imagetoURI, + self.imagemetadata, DocConfig.metadatanamespaces, + foundlabel, comment, thetypes, predobjmap, "Video")) + for video in foundmedia["video"]: + self.imagetoURI[video] = {"uri": str(subject)} + f.write(self.templates["videotemplate"].replace("{{video}}", str(video))) + for type in curtypes: + if type in SPARQLUtils.lexicontypes: + LexiconPage().generatePageWidget(graph, subject, f, {}, False) + if type in PersonPage.pageWidgetConstraint(): + PersonPage().generatePageWidget(graph, subject, self.templates, f, True) + HTMLExporter.processCollectionPages(collections, graph, subject,self.templates, f) + if geojsonrep != None and "geocollection" not in collections: + self.geocache = GeometryViewPage().generatePageWidget(graph, self.templates, subject, f, uritotreeitem, + geojsonrep, predobjmap, self.geocache, + {"dateprops": dateprops, "timeobj": timeobj, + "epsgcode": epsgcode, + "epsgdefslink": epsgdefslink, + "checkdepth": checkdepth, + "hasnonnslen": len(hasnonns)}) + elif "geocollection" in collections or nonns: + self.geocache = GeometryViewPage().generateCollectionWidget(graph, self.templates, subject, f, + uritotreeitem, + self.featurecollectionspaths, + {"completesavepath": completesavepath, + "nonns": nonns, "hasnonns": hasnonns, + "foundlabel": foundlabel, + "localOptimized": self.localOptimized, + "dateprops": dateprops, + "timeobj": timeobj, + "geocache": self.geocache, + "epsgcode": epsgcode, + "epsgdefslink": epsgdefslink, + "checkdepth": checkdepth, + "hasnonnslen": len(hasnonns)}) + f.write(self.templates["htmltabletemplate"].replace("{{tablecontent}}", tablecontents)) + if metadatatablecontentcounter >= 0: + f.write("
    Metadata
    ") + f.write(self.templates["htmltabletemplate"].replace("{{tablecontent}}", metadatatablecontents)) + tempfoot = self.replaceStandardVariables(self.templates["footer"], "", checkdepth, "false").replace( + "{{exports}}", + myexports).replace( + "{{license}}", curlicense).replace("{{bibtex}}", "").replace("{{stats}}", "") + tempfoot = DocUtils.conditionalArrayReplace(tempfoot, + [True, self.apis["ogcapifeatures"], self.apis["iiif"], self.apis["ckan"]], + [ + "[SPARQL] ", + "[OGC API Features] ", + "[IIIF] ", + "[CKAN]" + ], "{{apis}}") + f.write(tempfoot) + f.close() + return [postprocessing, nonnsmap] + + @staticmethod + def processCollectionPages(pagesmap, graph, subject, templates, f): + if "observationcollection" in pagesmap: + ObservationPage().generateCollectionWidget(graph, templates, subject, f) + if "lexicon" in pagesmap: + LexiconPage().generateCollectionWidget(graph, templates, subject, f) + + @staticmethod + def searchObjectConnectionsForAggregateData(graph, object, pred, geojsonrep, foundmedia, imageannos, + textannos, image3dannos, annobodies, label, unitlabel, nonns, inverse, + labellang, typeproperty, prefixes): + geoprop = False + annosource = None + incollection = False + if pred in SPARQLUtils.geopointerproperties: + geoprop = True + if pred in SPARQLUtils.collectionrelationproperties: + incollection = True + foundval = None + foundunit = None + tempvalprop = None + onelabel = None + bibtex = None + timeobj = None + for tup in graph.predicate_objects(object): + if str(tup[0]) in SPARQLUtils.labelproperties: + if tup[1].language == labellang: + label = str(tup[1]) + onelabel = str(tup[1]) + if pred == "http://www.w3.org/ns/oa#hasSelector" and tup[0] == URIRef(typeproperty) and ( + tup[1] == URIRef("http://www.w3.org/ns/oa#SvgSelector") or tup[1] == URIRef( + "http://www.w3.org/ns/oa#WKTSelector")): + for svglit in graph.objects(object, URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#value")): + if "" + thelabel + "" + else: + unitlabel = str(foundval) + " " + str(foundunit) + if pred == "http://www.w3.org/ns/oa#hasBody": + # print("ADD ANNO BODY: "+str({"value":foundval,"unit":foundunit,"type":"TextualBody","format":"text/plain"})) + annobodies.append({"value": foundval, "unit": foundunit, "type": "TextualBody", "format": "text/plain"}) + if foundunit is None and foundval is not None: + if "http" in foundval: + thelabel = DocUtils.getLabelForObject(str(foundunit), graph, prefixes) + unitlabel = "" + thelabel + "" + else: + unitlabel = str(foundval) + if pred == "http://www.w3.org/ns/oa#hasBody": + # print("ADD ANNO BODY: "+str({"value":foundval,"type":"TextualBody","format":"text/plain"})) + annobodies.append({"value": foundval, "type": "TextualBody", "format": "text/plain"}) + if annosource is not None: + for textanno in textannos: + textanno["src"] = annosource + for imganno in imageannos: + imganno["src"] = annosource + for imganno in image3dannos: + imganno["src"] = annosource + if label == "" and onelabel is not None: + label = onelabel + return {"geojsonrep": geojsonrep, "label": label, "unitlabel": unitlabel, "foundmedia": foundmedia, + "imageannos": imageannos, "textannos": textannos, "image3dannos": image3dannos, + "annobodies": annobodies, "bibtex": bibtex, "timeobj": timeobj} + + @staticmethod + def createHTMLTableValueEntry(subject, pred, object, ttlf, graph, baseurl, checkdepth, geojsonrep, foundmedia, + imageannos, textannos, image3dannos, annobodies, dateprops, inverse, nonns, labellang, + typeproperty, namespaceshort, generatePagesForNonNS, prefixes): + tablecontents = "" + label = "" + bibtex = None + timeobj = None + if isinstance(object, URIRef) or isinstance(object, BNode): + if ttlf is not None: + ttlf.add((subject, URIRef(pred), object)) + label = "" + unitlabel = "" + mydata = HTMLExporter.searchObjectConnectionsForAggregateData(graph, object, pred, geojsonrep, foundmedia, + imageannos, textannos, image3dannos, + annobodies, + label, unitlabel, nonns, inverse, labellang, + typeproperty, prefixes) + label = mydata["label"] + if label == "": + label = str(DocUtils.shortenURI(str(object))) + geojsonrep = mydata["geojsonrep"] + foundmedia = mydata["foundmedia"] + imageannos = mydata["imageannos"] + textannos = mydata["textannos"] + image3dannos = mydata["image3dannos"] + unitlabel = mydata["unitlabel"] + bibtex = mydata["bibtex"] + timeobj = mydata["timeobj"] + annobodies = mydata["annobodies"] + if inverse: + rdfares = " about=\"" + str(object) + "\" resource=\"" + str(subject) + "\"" + microdatares = " itemref=\"" + str(object) + "\" " + else: + rdfares = "resource=\"" + str(object) + "\"" + microdatares = " " + if baseurl in str(object) or isinstance(object, BNode): + rellink = DocUtils.generateRelativeLinkFromGivenDepth(baseurl, checkdepth, str(object), True) + tablecontents += "" + label + " (" + namespaceshort + ":" + str( + DocUtils.shortenURI(str(object))) + ")" + if bibtex is not None: + tablecontents += "
    [BIBTEX]
    " + str(bibtex) + "
    " + else: + res = DocUtils.replaceNameSpacesInLabel(prefixes, str(object)) + if res is not None: + tablecontents += "" + label + " (" + res["uri"] + ")" + else: + tablecontents += "" + label + "" + if bibtex is not None: + tablecontents += "
    [BIBTEX]
    " + str(bibtex) + "
    " + if generatePagesForNonNS: + rellink = DocUtils.generateRelativeLinkFromGivenDepth(str(baseurl), checkdepth, + str(baseurl) + "nonns_" + DocUtils.shortenURI( + str(object).replace(":", "_")), False) + tablecontents += " [x]" + if unitlabel != "": + tablecontents += " [" + str(unitlabel) + "]" + if timeobj is not None: + res = str(OWLTimePage.timeObjectToHTML(timeobj, prefixes)) + if res != "None": + tablecontents += " [" + str(res) + "]" + dateprops = timeobj + tablecontents += "    " + else: + label = str(object) + if ttlf is not None: + ttlf.add((subject, URIRef(pred), object)) + if isinstance(object, Literal) and object.datatype != None: + res = DocUtils.replaceNameSpacesInLabel(prefixes, str(object.datatype)) + objstring = str(object).replace("<", "<").replace(">", ">") + if str(object.datatype) == "http://www.w3.org/2001/XMLSchema#anyURI": + objstring = "" + str(object) + "" + if str(object.datatype) in SPARQLUtils.timeliteraltypes and dateprops != None and DocUtils.shortenURI( + str(pred), True) not in SPARQLUtils.metadatanamespaces and str(pred) not in dateprops: + dateprops.append(str(pred)) + if res != None: + tablecontents += "", ">").replace("\"", "'") + "\" datatype=\"" + str( + object.datatype) + "\">" + HTMLExporter.truncateValue( + objstring) + " (" + res["uri"] + ")" + else: + tablecontents += "", ">").replace("\"", "'") + "\" datatype=\"" + str( + object.datatype) + "\">" + HTMLExporter.truncateValue( + objstring) + " (" + DocUtils.shortenURI(str(object.datatype)) + ")" + geojsonrep = LiteralUtils.resolveGeoLiterals(URIRef(pred), object, graph, geojsonrep, nonns, subject) + else: + if object.language is not None: + tablecontents += "", ">").replace("\"", + "'") + "\" datatype=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#langString\" xml:lang=\"" + str( + object.language) + "\">" + HTMLExporter.truncateValue( + str(object).replace("<", "<").replace(">", + ">")) + " (data:langString) (iso6391:" + str( + object.language) + ")" + else: + tablecontents += HTMLExporter.detectStringLiteralContent(pred, object) + return {"html": tablecontents, "geojson": geojsonrep, "foundmedia": foundmedia, "imageannos": imageannos, + "textannos": textannos, "image3dannos": image3dannos, "annobodies": annobodies, "label": label, + "timeobj": dateprops} + + @staticmethod + def processLicense(license): + if license == None or license == "" or license == "No License Statement": + return ["", None] + if license.startswith("CC"): + spl = license.split(" ") + res = """This work is released under + Creative Commons License""" + licenseuri = "http://creativecommons.org/licenses/" + str(spl[1]).lower() + "/" + str(spl[2]) + return [res, licenseuri] + else: + return ["""All rights reserved.""", None] + + @staticmethod + def formatPredicate(tup, baseurl, checkdepth, tablecontents, graph, reverse, labellang, prefixes): + label = DocUtils.getLabelForObject(URIRef(str(tup)), graph, prefixes, labellang) + tablecontents += "" + if reverse: + tablecontents += "Is " + if baseurl in str(tup): + rellink = DocUtils.generateRelativeLinkFromGivenDepth(baseurl, checkdepth, str(tup), True) + tablecontents += "" + label + "" + else: + res = DocUtils.replaceNameSpacesInLabel(prefixes, tup) + if res != None: + tablecontents += "" + label + " (" + res[ + "uri"] + ") " + else: + tablecontents += "" + label + " " + tablecontents += "" + if reverse: + tablecontents += " of" + tablecontents += "" + return tablecontents + + @staticmethod + def truncateValue(value, limit=150): + if len(value) > limit: + return "
    " + value[0:limit] + " (...)" + str( + value[limit:]) + "
    " + return value + + @staticmethod + def detectStringLiteralContent(pred, object): + if object.startswith("http://") or object.startswith("https://"): + return "" + str( + object) + " (xsd:string)" + elif object.startswith("www."): + return "http://" + str( + object) + " (xsd:string)" + elif re.search(r'(10[.][0-9]{2,}(?:[.][0-9]+)*/(?:(?![%"#? ])\\S)+)', str(object)): + return "" + str( + object) + " (xsd:anyURI)" + elif re.search(r'[\w.]+\@[\w.]+', object): + return "mailto:" + str( + object) + " (xsd:string)" + return "", + ">").replace( + "\"", "'") + "\" datatype=\"http://www.w3.org/2001/XMLSchema#string\">" + str(object).replace("<", + "<").replace( + ">", + ">") + " (xsd:string)" diff --git a/util/export/data/exporter/rdf/miscexporter.py b/util/export/data/miscexporter.py similarity index 98% rename from util/export/data/exporter/rdf/miscexporter.py rename to util/export/data/miscexporter.py index 0ba0dc461..4b111187c 100644 --- a/util/export/data/exporter/rdf/miscexporter.py +++ b/util/export/data/miscexporter.py @@ -1,7 +1,7 @@ import json import os -from .....doc.docutils import DocUtils +from ...doc.docutils import DocUtils class MiscExporter: diff --git a/util/export/data/voidexporter.py b/util/export/data/voidexporter.py new file mode 100644 index 000000000..246676bef --- /dev/null +++ b/util/export/data/voidexporter.py @@ -0,0 +1,175 @@ +from rdflib import URIRef, Literal, Graph +from ...doc.docconfig import DocConfig + +from ...doc.docutils import DocUtils + + +class VoidExporter: + + @staticmethod + def createVoidDataset(pubconfig, licenseuri, stats, subjectstorender, classtree=None, propstats=None, + nonnscount=None, nscount=None, objectmap=None): + g = Graph() + g.bind("voaf", "http://purl.org/vocommons/voaf#") + g.bind("vext", "http://ldf.fi/void-ext#") + g.bind("vann", "http://purl.org/vocab/vann/") + g.bind("adms", "http://www.w3.org/ns/adms#") + g.bind("dcat", "http://www.w3.org/ns/dcat#") + dsname = pubconfig["datasettitle"] + if pubconfig["datasettitle"] == None or pubconfig["datasettitle"] == "": + dsname = "dataset" + dsname = dsname.replace(" ", "_") + voidds = pubconfig["prefixnamespace"] + dsname + if pubconfig["repository"] is not None and pubconfig["repository"] != "" and pubconfig["repository"].startswith( + "http"): + g.add((URIRef(pubconfig["repository"]), URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + URIRef("http://www.w3.org/ns/adms#AssetRepository"))) + g.add((URIRef(pubconfig["repository"]), URIRef("http://www.w3.org/ns/dcat#accessURL"), + Literal(str(pubconfig["repository"]), datatype="http://www.w3.org/2001/XMLSchema#anyURI"))) + g.add((URIRef(pubconfig["repository"]), URIRef("http://www.w3.org/ns/dcat#dataset"), + URIRef(voidds))) + g.add((URIRef(pubconfig["repository"]), URIRef("http://www.w3.org/2000/01/rdf-schema#label"), + Literal("Repository for " + str(dsname), lang="en"))) + g.add((URIRef(voidds), URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + URIRef("http://rdfs.org/ns/void#Dataset"))) + g.add((URIRef(voidds), URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + URIRef("http://www.w3.org/ns/adms#Asset"))) + g.add((URIRef(voidds), URIRef("http://www.w3.org/2000/01/rdf-schema#label"), + Literal(dsname, lang="en"))) + g.add((URIRef(voidds), URIRef("http://purl.org/dc/terms/title"), + Literal(dsname, lang="en"))) + if pubconfig["labellang"] is not None and pubconfig["labellang"] != "": + g.add((URIRef(voidds), URIRef("http://purl.org/dc/elements/1.1/language"), + URIRef("http://www.lexvo.org/page/iso639-1/" + str(pubconfig["labellang"])))) + g.add((URIRef(voidds), URIRef("http://purl.org/dc/terms/modified"), + Literal(pubconfig["modtime"], datatype="http://www.w3.org/2001/XMLSchema#dateTime"))) + if licenseuri is not None: + g.add((URIRef(voidds), URIRef("http://purl.org/dc/terms/license"), + URIRef(licenseuri))) + g.add((URIRef(voidds), URIRef("http://rdfs.org/ns/void#dataDump"), + URIRef(pubconfig["deploypath"] + "/index.ttl"))) + g.add((URIRef(voidds), URIRef("http://xmlns.com/foaf/0.1/homepage"), + URIRef(pubconfig["deploypath"]))) + g.add((URIRef(voidds), URIRef("http://www.w3.org/ns/dcat#landingPage"), + URIRef(pubconfig["deploypath"]))) + g.add((URIRef(voidds), URIRef("http://xmlns.com/foaf/0.1/page"), + URIRef(pubconfig["deploypath"] + "/index.html"))) + g.add((URIRef(voidds), URIRef("http://rdfs.org/ns/void#dataDump"), + URIRef(pubconfig["deploypath"] + "/index.ttl"))) + g.add((URIRef(voidds), URIRef("http://www.w3.org/ns/dcat#distribution"), + URIRef(voidds + "_dist_ttl"))) + g.add((URIRef(voidds + "_dist_ttl"), URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + URIRef("http://www.w3.org/ns/adms#AssetDistribution"))) + g.add((URIRef(voidds + "_dist_ttl"), URIRef("http://www.w3.org/2000/01/rdf-schema#label"), + Literal(dsname + " TTL Distribution", lang="en"))) + g.add((URIRef(voidds + "_dist_ttl"), URIRef("http://www.w3.org/ns/dcat#downloadURL"), + Literal(pubconfig["deploypath"] + "/index.ttl", datatype="http://www.w3.org/2001/XMLSchema#anyURI"))) + g.add((URIRef(voidds + "_dist_ttl"), URIRef("http://www.w3.org/ns/dcat#mediaType"), + URIRef("http://www.w3.org/ns/formats/Turtle"))) + g.add((URIRef(voidds), URIRef("http://rdfs.org/ns/void#feature"), + URIRef("http://www.w3.org/ns/formats/Turtle"))) + g.add((URIRef(voidds), URIRef("http://rdfs.org/ns/void#feature"), + URIRef("http://www.w3.org/ns/formats/RDFa"))) + if pubconfig["startconcept"] is not None and pubconfig["startconcept"] != "": + g.add((URIRef(voidds), URIRef("http://rdfs.org/ns/void#rootResource"), + URIRef(pubconfig["startconcept"].replace("index.html", "")))) + g.add((URIRef(voidds), URIRef("http://rdfs.org/ns/void#exampleResource"), + URIRef(pubconfig["startconcept"].replace("index.html", "")))) + for stat in stats: + g.add((URIRef(voidds), URIRef(stat), + Literal(stats[stat], datatype="http://www.w3.org/2001/XMLSchema#integer"))) + g.add((URIRef(voidds), URIRef("http://rdfs.org/ns/void#uriSpace"), + Literal(pubconfig["prefixnamespace"], datatype="http://www.w3.org/2001/XMLSchema#string"))) + g.add((URIRef(voidds), URIRef("http://purl.org/vocab/vann/preferredNamespaceUri"), + Literal(pubconfig["prefixnamespace"], datatype="http://www.w3.org/2001/XMLSchema#anyURI"))) + g.add((URIRef(voidds), URIRef("http://purl.org/vocab/vann/preferredNamespacePrefix"), + Literal(pubconfig["namespaceshort"], datatype="http://www.w3.org/2001/XMLSchema#string"))) + for ns_prefix, namespace in g.namespaces(): + g.add((URIRef(voidds), URIRef("http://rdfs.org/ns/void#vocabulary"), URIRef(namespace))) + g.add((URIRef(namespace), URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + URIRef("http://purl.org/vocommons/voaf#Vocabulary"))) + if "nstolabel" in pubconfig["prefixes"] and str(namespace) in pubconfig["prefixes"]["nstolabel"]: + g.add((URIRef(namespace), URIRef("http://www.w3.org/2000/01/rdf-schema#label"), + Literal(pubconfig["prefixes"]["nstolabel"][str(namespace)], lang="en"))) + else: + g.add((URIRef(namespace), URIRef("http://www.w3.org/2000/01/rdf-schema#label"), + Literal(str(ns_prefix) + " Vocabulary", lang="en"))) + g.add((URIRef(namespace), URIRef("http://purl.org/vocab/vann/preferredNamespaceUri"), + Literal(namespace, datatype="http://www.w3.org/2001/XMLSchema#anyURI"))) + g.add((URIRef(namespace), URIRef("http://purl.org/vocab/vann/preferredNamespacePrefix"), + Literal(ns_prefix, datatype="http://www.w3.org/2001/XMLSchema#string"))) + g.add((URIRef(pubconfig["prefixnamespace"] + str(ns_prefix) + "_" + str(dsname) + "_occ"), + URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + URIRef("http://purl.org/vocommons/voaf#DatasetOccurrence"))) + g.add((URIRef(pubconfig["prefixnamespace"] + str(ns_prefix) + "_" + str(dsname) + "_occ"), + URIRef("http://www.w3.org/2000/01/rdf-schema#label"), + Literal("Occurrences of vocabulary " + str(namespace) + " in " + dsname))) + if nscount is not None and str(namespace) in nscount: + g.add((URIRef(pubconfig["prefixnamespace"] + str(ns_prefix) + "_" + str(dsname) + "_occ"), + URIRef("http://purl.org/vocommons/voaf#occurrences"), + Literal(str(nscount[str(namespace)]), datatype="http://www.w3.org/2001/XMLSchema#integer"))) + g.add((URIRef(namespace), URIRef("http://purl.org/vocommons/voaf#usageInDataset"), + URIRef(namespace + "_" + str(dsname) + "_occ"))) + g.add((URIRef(pubconfig["prefixnamespace"] + str(ns_prefix) + "_" + str(dsname) + "_occ"), + URIRef("http://purl.org/vocommons/voaf#inDataset"), URIRef(voidds))) + if str(namespace) in DocConfig.namespaceToTopic: + for entry in DocConfig.namespaceToTopic[str(namespace)]: + g.add((URIRef(voidds), URIRef("http://www.w3.org/ns/dcat#keyword"), + Literal(DocUtils.shortenURI(entry["uri"]).replace("_", " "), lang="en"))) + g.add((URIRef(voidds), URIRef("http://purl.org/dc/terms/subject"), URIRef(entry["uri"]))) + g.add((URIRef(entry["uri"]), URIRef("http://www.w3.org/2000/01/rdf-schema#label"), + Literal(entry["label"], lang="en"))) + for pred in propstats: + cururi = voidds + "_" + DocUtils.shortenURI(pred) + g.add((URIRef(voidds), URIRef("http://rdfs.org/ns/void#propertyPartition"), URIRef(cururi))) + g.add((URIRef(cururi), URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + URIRef("http://rdfs.org/ns/void#Dataset"))) + g.add((URIRef(cururi), URIRef("http://www.w3.org/2000/01/rdf-schema#label"), + Literal("Property Partition: " + str(DocUtils.shortenURI(pred)), lang="en"))) + g.add((URIRef(cururi), URIRef("http://rdfs.org/ns/void#property"), URIRef(pred))) + g.add((URIRef(cururi), URIRef("http://rdfs.org/ns/void#triples"), + Literal(str(propstats[pred]["triples"]), datatype="http://www.w3.org/2001/XMLSchema#integer"))) + subjectstorender.add(URIRef(cururi)) + for item in classtree["core"]["data"]: + if item["type"] == "class": + cururi = voidds + "_" + DocUtils.shortenURI(item["id"]) + g.add((URIRef(voidds), URIRef("http://rdfs.org/ns/void#classPartition"), URIRef(cururi))) + g.add((URIRef(cururi), URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + URIRef("http://rdfs.org/ns/void#Dataset"))) + g.add((URIRef(cururi), URIRef("http://www.w3.org/2000/01/rdf-schema#label"), + Literal("Class Partition: " + str(DocUtils.shortenURI(item["id"])), lang="en"))) + g.add((URIRef(cururi), URIRef("http://rdfs.org/ns/void#class"), URIRef(item["id"]))) + if item["id"] in objectmap: + g.add((URIRef(cururi), URIRef("http://rdfs.org/ns/void#entities"), + Literal(str(objectmap[item["id"]]), datatype="http://www.w3.org/2001/XMLSchema#integer"))) + # subjectstorender.add(URIRef(cururi)) + for prop in nonnscount: + for ns in nonnscount[prop]: + cururi = voidds + "_" + ns.replace("http://", "").replace("https://", "").replace("/", "_").replace("#", + "_") + g.add((URIRef(cururi), URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + URIRef("http://rdfs.org/ns/void#Linkset"))) + g.add((URIRef(cururi), URIRef("http://www.w3.org/2000/01/rdf-schema#label"), Literal( + "Linkset: " + str(DocUtils.shortenURI(voidds)) + " - " + str( + DocUtils.getLabelForObject(URIRef(ns), g, pubconfig["prefixes"])), lang="en"))) + g.add((URIRef(cururi), URIRef("http://rdfs.org/ns/void#subjectsTarget"), URIRef(voidds))) + g.add((URIRef(cururi), URIRef("http://rdfs.org/ns/void#objectsTarget"), URIRef(ns))) + g.add((URIRef(cururi), URIRef("http://rdfs.org/ns/void#linkPredicate"), URIRef(prop))) + g.add((URIRef(cururi), URIRef("http://rdfs.org/ns/void#triples"), + Literal(str(nonnscount[prop][ns]), datatype="http://www.w3.org/2001/XMLSchema#integer"))) + subjectstorender.add(URIRef(cururi)) + g.serialize(pubconfig["outpath"] + "/void.ttl", encoding="utf-8") + return {"graph": g, "subjects": subjectstorender} + + @staticmethod + def toHTML(stats, deploypath): + result = "
    Dataset Statistics [VOID]" + result += "" + for stat in stats: + result += "" + result += "
    PropertyValue
    " + str( + DocUtils.shortenURI(stat)) + "" + str(stats[ + stat]) + " (xsd:integer)
    " + return result \ No newline at end of file diff --git a/util/export/data/exporter/rdf/vowlexporter.py b/util/export/data/vowlexporter.py similarity index 98% rename from util/export/data/exporter/rdf/vowlexporter.py rename to util/export/data/vowlexporter.py index bce0b14d9..eea4b62d3 100644 --- a/util/export/data/exporter/rdf/vowlexporter.py +++ b/util/export/data/vowlexporter.py @@ -1,7 +1,6 @@ from rdflib import Graph, URIRef from qgis.core import Qgis, QgsMessageLog import json -from qgis.core import Qgis, QgsMessageLog class OWL2VOWL(): @@ -14,7 +13,7 @@ def getTypeForProperty(prop,graph,typeproperty): #print(tup) if str(tup)!="http://www.w3.org/1999/02/22-rdf-syntax-ns#Property": return OWL2VOWL.normalizeNS(str(tup)) - return "rdf:Property" + return "data:Property" @staticmethod def getBaseIRI(iri): @@ -30,7 +29,7 @@ def getIRILabel(iri): @staticmethod def normalizeNS(prop): - return prop.replace("http://www.w3.org/1999/02/22-rdf-syntax-ns#","rdf:").replace("http://www.w3.org/2000/01/rdf-schema#","rdfs:").replace("http://www.w3.org/2002/07/owl#","owl:") + return prop.replace("http://www.w3.org/1999/02/22-rdf-syntax-ns#","data:").replace("http://www.w3.org/2000/01/rdf-schema#","rdfs:").replace("http://www.w3.org/2002/07/owl#","owl:") #def inferDomainRanges(self,g,typeproperty): # for subj in g.subjects(): diff --git a/util/export/data/exporter/exporterutils.py b/util/export/exporterutils.py similarity index 91% rename from util/export/data/exporter/exporterutils.py rename to util/export/exporterutils.py index 64cf27d38..b9211dc10 100644 --- a/util/export/data/exporter/exporterutils.py +++ b/util/export/exporterutils.py @@ -1,6 +1,6 @@ -from .rdf.geoexporter import GeoExporter -from .rdf.graphexporter import GraphExporter -from .rdf.miscexporter import MiscExporter +from .data.geoexporter import GeoExporter +from .data.graphexporter import GraphExporter +from .data.miscexporter import MiscExporter class ExporterUtils: diff --git a/util/export/data/exporter/__init__.py b/util/export/layer/__init__.py similarity index 100% rename from util/export/data/exporter/__init__.py rename to util/export/layer/__init__.py diff --git a/util/export/data/exporter/layer/layerexporter.py b/util/export/layer/layerexporter.py similarity index 98% rename from util/export/data/exporter/layer/layerexporter.py rename to util/export/layer/layerexporter.py index 88fc35581..75eadfcf2 100644 --- a/util/export/data/exporter/layer/layerexporter.py +++ b/util/export/layer/layerexporter.py @@ -4,11 +4,10 @@ import re import urllib.parse -from ....srs.crsexporttools import ConvertCRS +from ...export.srs.crsexporttools import ConvertCRS from ..exporterutils import ExporterUtils -from .....layerutils import LayerUtils -from .....sparqlutils import SPARQLUtils -from qgis.core import Qgis,QgsTask, QgsMessageLog +from ...layerutils import LayerUtils +from ...sparqlutils import SPARQLUtils from rdflib import Graph @@ -348,7 +347,7 @@ def layerToGraphML(layer): "geom:asWkt") + "\n\n\n\n") edgeset.add("\n\n\n" + str( - "rdf:type") + "\n\n\n\n") + "data:type") + "\n\n\n\n") edgeset.add("\n\n\n" + str( @@ -356,7 +355,7 @@ def layerToGraphML(layer): edgecounter += 1 edgeset.add("\n\n\n" + str( - "rdf:type") + "\n\n\n\n") + "data:type") + "\n\n\n\n") literalcounter += 1 edgecounter += 1 fidcounter += 1 diff --git a/util/export/pages/indexviewpage.py b/util/export/pages/indexviewpage.py new file mode 100644 index 000000000..3d2ade0cf --- /dev/null +++ b/util/export/pages/indexviewpage.py @@ -0,0 +1,165 @@ +from rdflib import Graph, URIRef +from ...doc.docutils import DocUtils +from ..exporterutils import ExporterUtils +import re +import urllib.parse + +class IndexViewPage: + + + @staticmethod + def getAccessFromBaseURL(baseurl, savepath): + return savepath.replace(baseurl, "") + + + @staticmethod + def createIndexPages(pubconfig,templates,apis,paths,subjectstorender,uritotreeitem,voidds,tree,classlist,graph,voidstatshtml,curlicense): + indpcounter = 0 + print("PATHS: "+str(paths)) + print(tree) + for path in paths: + #if indpcounter % 10 == 0: + # DocUtils.updateProgressBar(indpcounter, len(paths), "Creating Index Pages") + subgraph = Graph(bind_namespaces="rdflib") + checkdepth = DocUtils.checkDepthFromPath(path, pubconfig["outpath"], path) - 1 + sfilelink = DocUtils.generateRelativeLinkFromGivenDepth(pubconfig["prefixnamespace"], checkdepth, + pubconfig["corpusid"] + '_search.js', False) + classtreelink = DocUtils.generateRelativeLinkFromGivenDepth(pubconfig["prefixnamespace"], checkdepth, + pubconfig["corpusid"] + "_classtree.js", False) + stylelink = DocUtils.generateRelativeLinkFromGivenDepth(pubconfig["prefixnamespace"], checkdepth, "style.css", False) + scriptlink = DocUtils.generateRelativeLinkFromGivenDepth(pubconfig["prefixnamespace"], checkdepth, "startscripts.js", + False) + proprelations = DocUtils.generateRelativeLinkFromGivenDepth(pubconfig["prefixnamespace"], checkdepth, + "proprelations.js", False) + epsgdefslink = DocUtils.generateRelativeLinkFromGivenDepth(pubconfig["prefixnamespace"], checkdepth, "epsgdefs.js", + False) + vowllink = DocUtils.generateRelativeLinkFromGivenDepth(pubconfig["prefixnamespace"], checkdepth, "vowl_result.js", + False) + nslink = pubconfig["prefixnamespace"] + str(IndexViewPage.getAccessFromBaseURL(str(pubconfig["outpath"]), str(path))) + for sub in subjectstorender: + if nslink in sub: + for tup in graph.predicate_objects(sub): + subgraph.add((sub, tup[0], tup[1])) + if apis["solidexport"]: + subgraph.add((URIRef(sub.replace("nslink", "")), + URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + URIRef("http://www.w3.org/ns/ldp#Container"))) + subgraph.add((URIRef(sub.replace("nslink", "")), + URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + URIRef("http://www.w3.org/ns/ldp#BasicContainer"))) + subgraph.add((URIRef(sub.replace("nslink", "")), + URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + URIRef("http://www.w3.org/ns/ldp#Resource"))) + for ex in pubconfig["exports"]: + if ex in ExporterUtils.exportToFunction: + if ex not in ExporterUtils.rdfformats: + with open(path + "index." + str(ex), 'w', encoding='utf-8') as f: + ExporterUtils.exportToFunction[ex](subgraph, f, subjectstorender, classlist, ex) + f.close() + else: + ExporterUtils.exportToFunction[ex](subgraph, path + "index." + str(ex), subjectstorender, + classlist, ex) + relpath = DocUtils.generateRelativePathFromGivenDepth(checkdepth) + print("RELPATH: " + str(relpath)) + indexhtml = DocUtils.replaceStandardVariables(templates["htmltemplate"], voidds, checkdepth, + str(nslink == pubconfig["prefixnamespace"]).lower(),pubconfig) + indexhtml = indexhtml.replace("{{iconprefixx}}", + (relpath + "icons/" if pubconfig["offlinecompat"] else "")).replace("{{baseurl}}", + pubconfig["prefixnamespace"]).replace( + "{{relativedepth}}", str(checkdepth)).replace("{{relativepath}}", relpath).replace("{{toptitle}}", + "Index page for " + nslink).replace( + "{{title}}", "Index page for " + str(nslink) + "").replace("{{startscriptpath}}", scriptlink).replace( + "{{stylepath}}", stylelink).replace("{{vowlpath}}", vowllink) \ + .replace("{{classtreefolderpath}}", classtreelink).replace("{{baseurlhtml}}", nslink).replace( + "{{proprelationpath}}", proprelations).replace("{{nonnslink}}", "").replace("{{scriptfolderpath}}", + sfilelink).replace( + "{{exports}}", templates["nongeoexports"]).replace("{{bibtex}}", "").replace("{{subjectencoded}}", + urllib.parse.quote( + str(voidds))) + indexhtml += "

    This page shows information about linked data resources in HTML. Choose the classtree navigation or search to browse the data

    " + \ + templates["vowltemplate"].replace("{{vowlpath}}", "minivowl_result.js") + if pubconfig["startconcept"] is not None and path == pubconfig["outpath"] and pubconfig["startconcept"] in uritotreeitem: + startconcept=pubconfig["startconcept"] + if pubconfig["createColl"]: + indexhtml += "

    Start exploring the graph here: \""" + DocUtils.shortenURI( + startconcept) + "

    " + else: + indexhtml += "

    Start exploring the graph here: \""" + DocUtils.shortenURI( + startconcept) + "

    " + indexhtml += "" + for item in tree["core"]["data"]: + if (item["type"] == "geoclass" or item["type"] == "class" or item["type"] == "featurecollection" or + item["type"] == "geocollection") and "instancecount" in item and item["instancecount"] > 0: + exitem = None + for item2 in tree["core"]["data"]: + if item2["parent"] == item["id"] and ( + item2["type"] == "instance" or item2["type"] == "geoinstance") and nslink in item2[ + "id"]: + checkdepth = DocUtils.checkDepthFromPath(path, pubconfig["prefixnamespace"], item2["id"]) - 1 + exitem = "" + break + if exitem != None: + if pubconfig["createColl"]: + indexhtml += "" + else: + indexhtml += "" + indexhtml += "" + exitem + "" + indexhtml += "
    ClassNumber of instancesInstance Example
    \""" + str(item2["text"]) + "
    \""" + str( + item["text"]) + "
    \""" + str(item["text"]) + "" + str( + item["instancecount"]) + "
    " + tempfoot = DocUtils.replaceStandardVariables(templates["footer"], "", checkdepth, + str(nslink == pubconfig["prefixnamespace"]).lower(),pubconfig).replace( + "{{license}}", curlicense).replace("{{exports}}", templates["nongeoexports"]).replace("{{bibtex}}", + "").replace( + "{{stats}}", voidstatshtml) + tempfoot = DocUtils.conditionalArrayReplace(tempfoot, [True, apis["ogcapifeatures"], apis["iiif"], + apis["ckan"]], + [ + "[SPARQL] ", + "[OGC API Features] ", + "[IIIF] ", + "[CKAN]" + ], "{{apis}}") + indexhtml += tempfoot + # print(path) + with open(path + "index.html", 'w', encoding='utf-8') as f: + f.write(indexhtml) + f.close() \ No newline at end of file diff --git a/util/export/pages/owltimepage.py b/util/export/pages/owltimepage.py index fe688d0a6..0e8916646 100644 --- a/util/export/pages/owltimepage.py +++ b/util/export/pages/owltimepage.py @@ -1,5 +1,5 @@ -from doc.docconfig import DocConfig -from doc.docutils import DocUtils +from ...doc.docconfig import DocConfig +from ...doc.docutils import DocUtils from rdflib import URIRef, Literal class OWLTimePage: diff --git a/util/export/pages/page.py b/util/export/pages/page.py new file mode 100644 index 000000000..82863a270 --- /dev/null +++ b/util/export/pages/page.py @@ -0,0 +1,22 @@ +from abc import ABC,abstractmethod + +class Page(ABC): + @abstractmethod + def pageWidgetConstraint(self): + pass + + @abstractmethod + def collectionConstraint(self): + pass + + @abstractmethod + def resolveBibtexReference(predobjs, item, graph): + pass + + @abstractmethod + def generatePageWidget(self, graph, memberid, templates, f, pageWidget=False): + pass + + @abstractmethod + def generateCollectionWidget(self, graph, templates, subject, f): + pass \ No newline at end of file diff --git a/util/export/srs/crsexporttools.py b/util/export/srs/crsexporttools.py index aaa951ef4..108be9849 100644 --- a/util/export/srs/crsexporttools.py +++ b/util/export/srs/crsexporttools.py @@ -132,7 +132,7 @@ class ConvertCRS: def __init__(self): - self.ttlhead = "@prefix rdf: .\n" + self.ttlhead = "@prefix data: .\n" self.ttlhead += "@prefix rdfs: .\n" self.ttlhead += "@prefix owl: .\n" self.ttlhead += "@prefix xsd: .\n" @@ -194,38 +194,38 @@ def crsToTTL(ttl,curcrs,x,geodcounter,crsclass): epsgcode=str(x) wkt=curcrs.to_wkt().replace("\"","'").strip() if crsclass is not None: - ttl.add("geoepsg:"+epsgcode+" rdf:type "+crsclass+" .\n") + ttl.add("geoepsg:"+epsgcode+" data:type "+crsclass+" .\n") elif "Projected CRS" in curcrs.type_name: - ttl.add("geoepsg:"+epsgcode+" rdf:type geocrs:ProjectedCRS .\n") + ttl.add("geoepsg:"+epsgcode+" data:type geocrs:ProjectedCRS .\n") elif "Geographic 2D CRS" in curcrs.type_name: - ttl.add("geoepsg:"+epsgcode+" rdf:type geocrs:GeographicCRS .\n") + ttl.add("geoepsg:"+epsgcode+" data:type geocrs:GeographicCRS .\n") elif "Geographic 3D CRS" in curcrs.type_name: - ttl.add("geoepsg:"+epsgcode+" rdf:type geocrs:GeographicCRS .\n") + ttl.add("geoepsg:"+epsgcode+" data:type geocrs:GeographicCRS .\n") elif "Bound CRS" in curcrs.type_name: - ttl.add("geoepsg:"+epsgcode+" rdf:type geocrs:BoundCRS .\n") + ttl.add("geoepsg:"+epsgcode+" data:type geocrs:BoundCRS .\n") elif "Vertical CRS" in curcrs.type_name: - ttl.add("geoepsg:"+epsgcode+" rdf:type geocrs:VerticalCRS .\n") + ttl.add("geoepsg:"+epsgcode+" data:type geocrs:VerticalCRS .\n") elif "Geocentric CRS" in curcrs.type_name: - ttl.add("geoepsg:"+epsgcode+" rdf:type geocrs:GeocentricCRS .\n") + ttl.add("geoepsg:"+epsgcode+" data:type geocrs:GeocentricCRS .\n") elif "Geographic 3D CRS" in curcrs.type_name: - ttl.add("geoepsg:"+epsgcode+" rdf:type geocrs:GeographicCRS .\n") + ttl.add("geoepsg:"+epsgcode+" data:type geocrs:GeographicCRS .\n") elif "Compound CRS" in curcrs.type_name: - ttl.add("geoepsg:"+epsgcode+" rdf:type geocrs:CompoundCRS .\n") + ttl.add("geoepsg:"+epsgcode+" data:type geocrs:CompoundCRS .\n") for subcrs in curcrs.sub_crs_list: ttl.add("geoepsg:"+epsgcode+" geocrs:includesSRS geoepsg:"+str(subcrs.to_epsg())+" .\n") else: - ttl.add("geoepsg:"+epsgcode+" rdf:type geocrs:CRS .\n") - ttl.add("geoepsg:"+epsgcode+" rdf:type prov:Entity. \n") + ttl.add("geoepsg:"+epsgcode+" data:type geocrs:CRS .\n") + ttl.add("geoepsg:"+epsgcode+" data:type prov:Entity. \n") ttl.add("geoepsg:"+epsgcode+" geocrs:isApplicableTo geocrsisbody:Earth .\n") - ttl.add("geoepsg:"+epsgcode+" rdf:type owl:NamedIndividual .\n") + ttl.add("geoepsg:"+epsgcode+" data:type owl:NamedIndividual .\n") ttl.add("geoepsg:"+epsgcode+" rdfs:label \""+curcrs.name.strip()+"\"@en .\n") ttl.add("geoepsg:"+epsgcode+" geocrs:isBound \""+str(curcrs.is_bound).lower()+"\"^^xsd:boolean . \n") if curcrs.coordinate_system is not None and curcrs.coordinate_system.name in coordinatesystem: - ttl.add("geoepsg:"+epsgcode+"_cs rdf:type "+coordinatesystem[curcrs.coordinate_system.name]+" . \n") + ttl.add("geoepsg:"+epsgcode+"_cs data:type "+coordinatesystem[curcrs.coordinate_system.name]+" . \n") if len(curcrs.coordinate_system.axis_list)==2: - ttl.add("geoepsg:"+epsgcode+"_cs rdf:type geocrs:PlanarCoordinateSystem . \n") + ttl.add("geoepsg:"+epsgcode+"_cs data:type geocrs:PlanarCoordinateSystem . \n") elif len(curcrs.coordinate_system.axis_list)==3: - ttl.add("geoepsg:"+epsgcode+"_cs rdf:type geocrs:3DCoordinateSystem . \n") + ttl.add("geoepsg:"+epsgcode+"_cs data:type geocrs:3DCoordinateSystem . \n") ttl.add("geoepsg:"+epsgcode+"_cs rdfs:label \"EPSG:"+epsgcode+" CS: "+curcrs.coordinate_system.name+"\" . \n") if curcrs.coordinate_system.remarks is not None: ttl.add("geoepsg:"+epsgcode+"_cs rdfs:comment \""+str(curcrs.coordinate_system.remarks)+"\"@en . \n") @@ -234,13 +234,13 @@ def crsToTTL(ttl,curcrs,x,geodcounter,crsclass): for axis in curcrs.coordinate_system.axis_list: axisid=axis.name.replace(" ","_").replace("(","_").replace(")","_").replace("/","_").replace("'","_")+"_"+axis.unit_name.replace(" ","_").replace("(","_").replace(")","_").replace("/","_").replace("'","_")+"_"+axis.direction.replace(" ","_").replace("(","_").replace(")","_").replace("/","_").replace("'","_") ttl.add("geoepsg:"+epsgcode+"_cs geocrs:axis geocrsaxis:"+axisid+" . \n") - ttl.add("geocrsaxis:"+axisid+" rdf:type geocrs:CoordinateSystemAxis . \n") + ttl.add("geocrsaxis:"+axisid+" data:type geocrs:CoordinateSystemAxis . \n") ttl.add("geocrsaxis:"+axisid+" geocrs:direction geocrs:"+axis.direction+" . \n") ttl.add("geocrsaxis:"+axisid+" geocrs:abbreviation \""+str(axis.abbrev).replace("\"","'")+"\"^^xsd:string . \n") ttl.add("geocrsaxis:"+axisid+" geocrs:unit_conversion_factor \""+str(axis.unit_conversion_factor)+"\"^^xsd:double . \n") ttl.add("geocrsaxis:"+axisid+" geocrs:unit_auth_code \""+str(axis.unit_auth_code)+"\"^^xsd:string . \n") ttl.add("geocrsaxis:"+axisid+" geocrs:unit_code \""+str(axis.unit_code)+"\"^^xsd:string . \n") - ttl.add("geocrsaxis:"+axis.direction+" rdf:type geocrs:AxisDirection . \n") + ttl.add("geocrsaxis:"+axis.direction+" data:type geocrs:AxisDirection . \n") if axis.unit_name in units: ttl.add("geocrsaxis:"+axisid+" geocrs:unit "+units[axis.unit_name]+" . \n") ttl.add("geocrsaxis:"+axisid+" rdfs:label \""+axis.name+" ("+str(units[axis.unit_name])+")\"@en . \n") @@ -268,7 +268,7 @@ def crsToTTL(ttl,curcrs,x,geodcounter,crsclass): ttl.add("geoepsg:"+epsgcode+" geocrs:scope \""+str(curcrs.scope).replace("\"","'")+"\"^^xsd:string . \n") if curcrs.area_of_use is not None: ttl.add("geoepsg:"+epsgcode+" geocrs:area_of_use geoepsg:"+epsgcode+"_area_of_use . \n") - ttl.add("geoepsg:"+epsgcode+"_area_of_use"+" rdf:type geocrs:AreaOfUse .\n") + ttl.add("geoepsg:"+epsgcode+"_area_of_use"+" data:type geocrs:AreaOfUse .\n") ttl.add("geoepsg:"+epsgcode+"_area_of_use"+" rdfs:label \""+str(curcrs.area_of_use.name).replace("\"","'")+"\"@en .\n") #b = box(curcrs.area_of_use.west, curcrs.area_of_use.south, curcrs.area_of_use.east, curcrs.area_of_use.north) #ttl.add("geoepsg:"+epsgcode+"_area_of_use"+" geocrs:extent \" "+str(b.wkt)+"\"^^geo:wktLiteral . \n") @@ -278,22 +278,22 @@ def crsToTTL(ttl,curcrs,x,geodcounter,crsclass): if curcrs.datum.ellipsoid is not None: if curcrs.datum.ellipsoid.name in spheroids: geoid=spheroids[curcrs.datum.ellipsoid.name] - ttl.add(geoid+" rdf:type geocrs:Ellipsoid . \n") + ttl.add(geoid+" data:type geocrs:Ellipsoid . \n") ttl.add(geoid+" rdfs:label \""+curcrs.datum.ellipsoid.name+"\"@en . \n") ttl.add(geoid+" geocrs:approximates geocrsisbody:Earth . \n") elif curcrs.get_geod().sphere: geoid="geocrsgeod:"+str(curcrs.datum.ellipsoid.name).replace(" ","_").replace("(","_").replace(")","_") - ttl.add(geoid+" rdf:type geocrs:Sphere . \n") + ttl.add(geoid+" data:type geocrs:Sphere . \n") ttl.add(geoid+" rdfs:label \""+curcrs.datum.ellipsoid.name+"\"@en . \n") ttl.add(geoid+" geocrs:approximates geocrsisbody:Earth . \n") else: geoid="geocrsgeod:"+str(curcrs.datum.ellipsoid.name).replace(" ","_").replace("(","_").replace(")","_") - ttl.add(geoid+" rdf:type geocrs:Geoid . \n") + ttl.add(geoid+" data:type geocrs:Geoid . \n") ttl.add(geoid+" rdfs:label \""+curcrs.datum.ellipsoid.name+"\"@en . \n") ttl.add(geoid+" geocrs:approximates geocrsisbody:Earth . \n") else: ttl.add("geoepsg:"+epsgcode+" geocrs:ellipsoid geocrsgeod:"+str(geodcounter)+" . \n") - ttl.add("geocrsgeod:geod"+str(geodcounter)+" rdf:type geocrs:Geoid . \n") + ttl.add("geocrsgeod:geod"+str(geodcounter)+" data:type geocrs:Geoid . \n") ttl.add(geoid+" rdfs:label \"Geoid "+str(geodcounter)+"\"@en . \n") ttl.add(geoid+" geocrs:approximates geocrsisbody:Earth . \n") ttl.add(geoid+" skos:definition \""+str(curcrs.get_geod().initstring)+"\"^^xsd:string . \n") @@ -318,7 +318,7 @@ def crsToTTL(ttl,curcrs,x,geodcounter,crsclass): ttl.add("geocrsoperation:"+str(coordoperationid)+" geocrs:has_ballpark_transformation \""+str(curcrs.coordinate_operation.has_ballpark_transformation)+"\"^^xsd:boolean . \n") if curcrs.coordinate_operation.area_of_use is not None: ttl.add("geocrsoperation:"+str(coordoperationid)+" geocrs:area_of_use geocrsaou:"+str(coordoperationid)+"_area_of_use . \n") - ttl.add("geocrsaou:"+str(coordoperationid)+"_area_of_use"+" rdf:type geocrs:AreaOfUse .\n") + ttl.add("geocrsaou:"+str(coordoperationid)+"_area_of_use"+" data:type geocrs:AreaOfUse .\n") ttl.add("geocrsaou:"+str(coordoperationid)+"_area_of_use"+" rdfs:label \""+str(curcrs.coordinate_operation.area_of_use.name).replace("\"","'")+"\"@en .\n") #b = box(curcrs.coordinate_operation.area_of_use.west, curcrs.coordinate_operation.area_of_use.south, curcrs.coordinate_operation.area_of_use.east, curcrs.coordinate_operation.area_of_use.north) #ttl.add("geocrsaou:"+str(coordoperationid)+"_area_of_use geocrs:extent \" "+str(b.wkt)+"\"^^geo:wktLiteral . \n") @@ -326,14 +326,14 @@ def crsToTTL(ttl,curcrs,x,geodcounter,crsclass): if curcrs.coordinate_operation.towgs84 is not None: print(curcrs.coordinate_operation.towgs84) for par in curcrs.coordinate_operation.params: - ttl.add(" geocrs:"+str(par.name)[0].lower()+str(par.name).title().replace(" ","")[1:]+" rdf:type owl:DatatypeProperty . \n") + ttl.add(" geocrs:"+str(par.name)[0].lower()+str(par.name).title().replace(" ","")[1:]+" data:type owl:DatatypeProperty . \n") ttl.add(" geocrs:"+str(par.name)[0].lower()+str(par.name).title().replace(" ","")[1:]+" rdfs:range xsd:double . \n") ttl.add(" geocrs:"+str(par.name)[0].lower()+str(par.name).title().replace(" ","")[1:]+" rdfs:domain geocrs:CoordinateOperation . \n") ttl.add(" geocrs:"+str(par.name)[0].lower()+str(par.name).title().replace(" ","")[1:]+" rdfs:label \""+str(par.name)+"\"@en . \n") ttl.add("geocrsoperation:"+str(coordoperationid)+" geocrs:"+str(par.name)[0].lower()+str(par.name).title().replace(" ","")[1:]+" \""+str(par.value)+"\"^^xsd:double . \n") for grid in curcrs.coordinate_operation.grids: ttl.add("geocrsoperation:"+str(coordoperationid)+" geocrs:grid geocrsgrid:"+str(grid.name).replace(" ","_")+" . \n") - ttl.add("geocrsgrid:"+str(grid.name).replace(" ","_")+" rdf:type geocrs:Grid . \n") + ttl.add("geocrsgrid:"+str(grid.name).replace(" ","_")+" data:type geocrs:Grid . \n") ttl.add("geocrsgrid:"+str(grid.name).replace(" ","_")+" rdfs:label \""+str(grid.full_name)+"\"@en . \n") ttl.add("geocrsgrid:"+str(grid.name).replace(" ","_")+" rdfs:label \""+str(grid.short_name)+"\"@en . \n") ttl.add("geocrsgrid:"+str(grid.name).replace(" ","_")+" geocrs:open_license \""+str(grid.open_license)+"\"^^xsd:boolean . \n") @@ -342,37 +342,37 @@ def crsToTTL(ttl,curcrs,x,geodcounter,crsclass): for operation in curcrs.coordinate_operation.operations: ttl.add("geocrsoperation:"+str(coordoperationid)+" geocrs:operation \""+str(operation).replace("\n","").replace("\"","'")+"\"^^xsd:string . \n") if curcrs.coordinate_operation.type_name==None: - ttl.add("geocrsoperation:"+str(coordoperationid)+" rdf:type geocrs:CoordinateOperation . \n") + ttl.add("geocrsoperation:"+str(coordoperationid)+" data:type geocrs:CoordinateOperation . \n") elif curcrs.coordinate_operation.type_name=="Conversion": found=False if curcrs.coordinate_operation.to_proj4() is not None: proj4string=curcrs.coordinate_operation.to_proj4().strip().replace("\"","'").replace("\n","") for prj in projections: if prj in proj4string: - ttl.add("geocrsoperation:"+str(coordoperationid)+" rdf:type "+projections[prj]+" . \n") + ttl.add("geocrsoperation:"+str(coordoperationid)+" data:type "+projections[prj]+" . \n") found=True break if not found: - ttl.add("geocrsoperation:"+str(coordoperationid)+" rdf:type geocrs:CoordinateConversionOperation . \n") + ttl.add("geocrsoperation:"+str(coordoperationid)+" data:type geocrs:CoordinateConversionOperation . \n") elif curcrs.coordinate_operation.type_name=="Transformation": - ttl.add("geocrsoperation:"+str(coordoperationid)+" rdf:type geocrs:CoordinateTransformationOperation . \n") + ttl.add("geocrsoperation:"+str(coordoperationid)+" data:type geocrs:CoordinateTransformationOperation . \n") elif curcrs.coordinate_operation.type_name=="Concatenated Operation": - ttl.add("geocrsoperation:"+str(coordoperationid)+" rdf:type geocrs:CoordinateConcatenatedOperation . \n") + ttl.add("geocrsoperation:"+str(coordoperationid)+" data:type geocrs:CoordinateConcatenatedOperation . \n") elif curcrs.coordinate_operation.type_name=="Other Coordinate Operation": - ttl.add("geocrsoperation:"+str(coordoperationid)+" rdf:type geocrs:OtherCoordinateOperation . \n") + ttl.add("geocrsoperation:"+str(coordoperationid)+" data:type geocrs:OtherCoordinateOperation . \n") ttl.add("geocrsoperation:"+str(coordoperationid)+" rdfs:label \""+curcrs.coordinate_operation.name+": "+curcrs.coordinate_operation.method_name+"\"@en . \n") if curcrs.datum is not None: datumid=str(curcrs.datum.name.replace(" ","_").replace("(","_").replace(")","_").replace("/","_").replace("'","_").replace("+","_plus").replace("[","_").replace("]","_")) ttl.add("geoepsg:"+epsgcode+" geocrs:datum geocrsdatum:"+str(datumid)+" . \n") if "Geodetic Reference Frame" in curcrs.datum.type_name: - ttl.add("geocrsdatum:"+str(datumid)+" rdf:type geocrs:GeodeticReferenceFrame . \n") + ttl.add("geocrsdatum:"+str(datumid)+" data:type geocrs:GeodeticReferenceFrame . \n") elif "Dynamic Vertical Reference Frame" in curcrs.datum.type_name: - ttl.add("geocrsdatum:"+str(datumid)+" rdf:type geocrs:DynamicVerticalReferenceFrame . \n") + ttl.add("geocrsdatum:"+str(datumid)+" data:type geocrs:DynamicVerticalReferenceFrame . \n") elif "Vertical Reference Frame" in curcrs.datum.type_name: - ttl.add("geocrsdatum:"+str(datumid)+" rdf:type geocrs:VerticalReferenceFrame . \n") + ttl.add("geocrsdatum:"+str(datumid)+" data:type geocrs:VerticalReferenceFrame . \n") else: print(curcrs.datum.type_name) - ttl.add("geocrsdatum:"+str(datumid)+" rdf:type geocrs:Datum . \n") + ttl.add("geocrsdatum:"+str(datumid)+" data:type geocrs:Datum . \n") ttl.add("geocrsdatum:"+str(datumid)+" rdfs:label \"Datum: "+curcrs.datum.name+"\"@en . \n") if curcrs.datum.remarks is not None: ttl.add("geocrsdatum:"+str(datumid)+" rdfs:comment \""+str(curcrs.datum.remarks)+"\"@en . \n") @@ -390,7 +390,7 @@ def crsToTTL(ttl,curcrs,x,geodcounter,crsclass): if curcrs.datum.ellipsoid is not None and curcrs.datum.ellipsoid.name in spheroids: ttl.add("geocrsdatum:"+str(datumid)+" geocrs:ellipse "+spheroids[curcrs.datum.ellipsoid.name]+" . \n") ttl.add(spheroids[curcrs.datum.ellipsoid.name]+" rdfs:label \""+str(curcrs.datum.ellipsoid.name)+"\"@en . \n") - ttl.add(spheroids[curcrs.datum.ellipsoid.name]+" rdf:type geocrs:Ellipsoid .\n") + ttl.add(spheroids[curcrs.datum.ellipsoid.name]+" data:type geocrs:Ellipsoid .\n") ttl.add(spheroids[curcrs.datum.ellipsoid.name]+" geocrs:inverse_flattening \""+str(curcrs.datum.ellipsoid.inverse_flattening)+"\"^^xsd:double .\n") if curcrs.datum.ellipsoid.remarks is not None: ttl.add(spheroids[curcrs.datum.ellipsoid.name]+" rdfs:comment \""+str(curcrs.datum.ellipsoid.remarks)+"\"^^xsd:string .\n") @@ -399,12 +399,12 @@ def crsToTTL(ttl,curcrs,x,geodcounter,crsclass): ttl.add("geocrsdatum:"+str(datumid)+" geocrs:ellipse \""+curcrs.datum.ellipsoid.name+"\" . \n") if curcrs.prime_meridian is not None: ttl.add("geocrsdatum:"+str(datumid)+" geocrs:primeMeridian geocrsmeridian:"+curcrs.prime_meridian.name.replace(" ","")+" . \n") - ttl.add("geocrsmeridian:"+curcrs.prime_meridian.name.replace(" ","")+" rdf:type geocrs:PrimeMeridian . \n") + ttl.add("geocrsmeridian:"+curcrs.prime_meridian.name.replace(" ","")+" data:type geocrs:PrimeMeridian . \n") ttl.add("geocrsmeridian:"+curcrs.prime_meridian.name.replace(" ","")+" rdfs:label \""+curcrs.prime_meridian.name+"\"@en . \n") ttl.add("geocrsmeridian:"+curcrs.prime_meridian.name.replace(" ","")+" geocrs:longitude \""+str(curcrs.prime_meridian.longitude)+"\"^^xsd:double . \n") if curcrs.prime_meridian.unit_name in units: ttl.add("geocrsmeridian:"+curcrs.prime_meridian.name.replace(" ","")+" geocrs:unit om:"+units[curcrs.prime_meridian.unit_name]+" . \n") - ttl.add(units[curcrs.prime_meridian.unit_name]+" rdf:type om:Unit .\n") + ttl.add(units[curcrs.prime_meridian.unit_name]+" data:type om:Unit .\n") else: ttl.add("geocrsmeridian:"+curcrs.prime_meridian.name.replace(" ","")+" geocrs:unit \""+str(curcrs.prime_meridian.unit_name)+"\" . \n") ttl.add("geocrsmeridian:"+curcrs.prime_meridian.name.replace(" ","")+" geocrs:asWKT \""+str(curcrs.prime_meridian.to_wkt()).replace("\"","'").replace("\n","")+"\" . \n") diff --git a/util/graphutils.py b/util/graphutils.py index 145532c42..e0e244b7d 100644 --- a/util/graphutils.py +++ b/util/graphutils.py @@ -16,7 +16,7 @@ class GraphUtils: "hasRDFSLabel": "PREFIX rdfs: ASK { ?a rdfs:label ?c . }", "hasSKOSPrefLabel": "PREFIX skos: ASK { ?a skos:prefLabel ?c . }", "hasDCTermsTitleLabel": "PREFIX dc: ASK { ?a dc:title ?c . }", - "hasRDFType": "ASK { ?a ?c . }", + "hasRDFType": "ASK { ?a ?c . }", "hassubClassOf": "ASK { ?a ?c . }", "hasSKOSTopConcept": "ASK { ?a ?c . }", "hasWKT": "PREFIX geosparql: ASK { ?a geosparql:asWKT ?c .}", @@ -67,7 +67,7 @@ def addDefaultConfigurationParameters(self,triplestorename,triplestoreurl,creden self.configuration["mandatoryvariables"] = [] self.configuration["active"] = True self.configuration["prefixes"] = {"owl": "http://www.w3.org/2002/07/owl#", - "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "data": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdfs": "http://www.w3.org/2000/01/rdf-schema#", "geosparql": "http://www.opengis.net/ont/geosparql#", "geof": "http://www.opengis.net/def/function/geosparql/", @@ -222,8 +222,8 @@ def detectTypeProperty(self,triplestoreurl,credentialUserName,credentialPassword results=SPARQLUtils.executeQuery(triplestoreurl,self.testQueries["hasRDFType"],{"auth":{"method":authmethod,"userCredential":credentialUserName,"userPassword":credentialPassword}}) if results!=False: if configuration!=None: - configuration["typeProperty"]="http:/www.w3.org/1999/02/22-rdf-syntax-ns#type" - return "http:/www.w3.org/1999/02/22-rdf-syntax-ns#type" + configuration["typeProperty"]="http:/www.w3.org/1999/02/22-data-syntax-ns#type" + return "http:/www.w3.org/1999/02/22-data-syntax-ns#type" return "" def detectSubClassOfProperty(self,triplestoreurl,credentialUserName,credentialPassword, authmethod,configuration=None): diff --git a/util/sparqlutils.py b/util/sparqlutils.py index f766b9a41..de3dc02b2 100644 --- a/util/sparqlutils.py +++ b/util/sparqlutils.py @@ -22,7 +22,7 @@ class SPARQLUtils: "http://www.opengis.net/ont/geosparql#dggsLiteral": "dggs" } - namespaces={"rdf":"http://www.w3.org/1999/02/22-rdf-syntax-ns#","rdfs":"http://www.w3.org/2000/01/rdf-schema#","owl":"http://www.w3.org/2002/07/owl#","dc":"http://purl.org/dc/terms/","skos":"http://www.w3.org/2004/02/skos/core#"} + namespaces={"data":"http://www.w3.org/1999/02/22-rdf-syntax-ns#","rdfs":"http://www.w3.org/2000/01/rdf-schema#","owl":"http://www.w3.org/2002/07/owl#","dc":"http://purl.org/dc/terms/","skos":"http://www.w3.org/2004/02/skos/core#"} annotationnamespaces=["http://www.w3.org/2004/02/skos/core#","http://www.w3.org/2000/01/rdf-schema#","http://purl.org/dc/terms/"] @@ -130,6 +130,8 @@ class SPARQLUtils: relationproperties={ "http://www.w3.org/2000/01/rdf-schema#seeAlso":"ObjectProperty", "http://www.w3.org/2000/01/rdf-schema#subClassOf": "ObjectProperty", + "http://www.w3.org/2004/02/skos/core#hasTopConcept": "ObjectProperty", + "http://www.w3.org/2004/02/skos/core#topConceptOf": "ObjectProperty", "http://www.w3.org/2004/02/skos/core#related": "ObjectProperty", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type": "ObjectProperty", "http://www.w3.org/2004/02/skos/core#exactMatch":"ObjectProperty", diff --git a/util/ui/uiutils.py b/util/ui/uiutils.py index cd0556b42..df218d395 100644 --- a/util/ui/uiutils.py +++ b/util/ui/uiutils.py @@ -289,7 +289,7 @@ def fillAttributeTable(queryresult,invprefixes,dlg,searchResultModel,nodetype,tr if "valtype" in queryresult[att]: itembutton.setText("Click to load samples... [" + str(queryresult[att]["valtype"]).replace( "http://www.w3.org/2001/XMLSchema#", "xsd:").replace("http://www.w3.org/1999/02/22-rdf-syntax-ns#", - "rdf:").replace( + "data:").replace( "http://www.opengis.net/ont/geosparql#", "geo:") + "]") itembutton.setData(str(queryresult[att]["valtype"]), UIUtils.dataslot_conceptURI) else: