From fbd85ae15423af8073d51ab335b7bad03fc8b9a3 Mon Sep 17 00:00:00 2001 From: david Date: Mon, 30 Sep 2024 08:26:47 +1000 Subject: [PATCH 1/7] MVP --- prez/app.py | 11 +- prez/config.py | 3 +- .../{ => base}/endpoint_metadata.ttl | 30 --- .../{ => base}/endpoint_nodeshapes.ttl | 59 ------ .../{ => features}/features_metadata.ttl | 0 .../{ => features}/features_nodeshapes.ttl | 0 .../routers/{ogc_router.py => base_router.py} | 44 ----- prez/routers/custom_endpoints.py | 163 ++++++++++++++++ prez/services/app_service.py | 22 ++- prez/services/connegp_service.py | 24 ++- prez/services/generate_endpoint_rdf.py | 178 ++++++++++++++++++ test_data/custom_endpoints.json | 55 ++++++ tests/custom_endpoints/conftest.py | 118 ++++++++++++ .../TO_FIX_test_dd_profiles.py | 0 .../TO_FIX_test_endpoints_vocprez.py | 0 .../TO_FIX_test_search.py | 0 tests/default_endpoints/__init__.py | 0 tests/{ => default_endpoints}/_test_count.py | 0 tests/{ => default_endpoints}/_test_cql.py | 0 .../_test_cql_fuseki.py | 0 .../_test_curie_generation.py | 0 tests/{ => default_endpoints}/conftest.py | 2 +- .../cql-fuseki-config.ttl | 0 .../test_alt_profiles.py | 0 tests/{ => default_endpoints}/test_bnode.py | 5 +- tests/{ => default_endpoints}/test_connegp.py | 0 .../{ => default_endpoints}/test_cql_time.py | 6 +- .../test_curie_endpoint.py | 0 .../test_endpoints_cache.py | 0 .../test_endpoints_catprez.py | 0 .../test_endpoints_concept_hierarchy.py | 0 .../test_endpoints_management.py | 0 .../test_endpoints_object.py | 0 .../test_endpoints_ok.py | 0 .../test_endpoints_profiles.py | 0 .../test_endpoints_spaceprez.py | 0 .../test_geojson_to_wkt.py | 0 .../test_node_selection_shacl.py | 12 +- tests/{ => default_endpoints}/test_ogc.py | 2 +- .../test_ogc_features_manual.py | 0 .../test_parse_datetimes.py | 0 .../test_property_selection_shacl.py | 0 .../test_query_construction.py | 0 .../test_redirect_endpoint.py | 0 .../test_remote_prefixes.py | 0 tests/{ => default_endpoints}/test_search.py | 0 tests/{ => default_endpoints}/test_sparql.py | 0 47 files changed, 569 insertions(+), 165 deletions(-) rename prez/reference_data/endpoints/{ => base}/endpoint_metadata.ttl (59%) rename prez/reference_data/endpoints/{ => base}/endpoint_nodeshapes.ttl (50%) rename prez/reference_data/endpoints/{ => features}/features_metadata.ttl (100%) rename prez/reference_data/endpoints/{ => features}/features_nodeshapes.ttl (100%) rename prez/routers/{ogc_router.py => base_router.py} (78%) create mode 100644 prez/routers/custom_endpoints.py create mode 100644 prez/services/generate_endpoint_rdf.py create mode 100644 test_data/custom_endpoints.json create mode 100755 tests/custom_endpoints/conftest.py rename tests/{ => default_endpoints}/TO_FIX_test_dd_profiles.py (100%) rename tests/{ => default_endpoints}/TO_FIX_test_endpoints_vocprez.py (100%) rename tests/{ => default_endpoints}/TO_FIX_test_search.py (100%) create mode 100644 tests/default_endpoints/__init__.py rename tests/{ => default_endpoints}/_test_count.py (100%) rename tests/{ => default_endpoints}/_test_cql.py (100%) rename tests/{ => default_endpoints}/_test_cql_fuseki.py (100%) rename tests/{ => default_endpoints}/_test_curie_generation.py (100%) rename tests/{ => default_endpoints}/conftest.py (97%) rename tests/{ => default_endpoints}/cql-fuseki-config.ttl (100%) rename tests/{ => default_endpoints}/test_alt_profiles.py (100%) rename tests/{ => default_endpoints}/test_bnode.py (87%) rename tests/{ => default_endpoints}/test_connegp.py (100%) rename tests/{ => default_endpoints}/test_cql_time.py (91%) rename tests/{ => default_endpoints}/test_curie_endpoint.py (100%) rename tests/{ => default_endpoints}/test_endpoints_cache.py (100%) rename tests/{ => default_endpoints}/test_endpoints_catprez.py (100%) rename tests/{ => default_endpoints}/test_endpoints_concept_hierarchy.py (100%) rename tests/{ => default_endpoints}/test_endpoints_management.py (100%) rename tests/{ => default_endpoints}/test_endpoints_object.py (100%) rename tests/{ => default_endpoints}/test_endpoints_ok.py (100%) rename tests/{ => default_endpoints}/test_endpoints_profiles.py (100%) rename tests/{ => default_endpoints}/test_endpoints_spaceprez.py (100%) rename tests/{ => default_endpoints}/test_geojson_to_wkt.py (100%) rename tests/{ => default_endpoints}/test_node_selection_shacl.py (76%) rename tests/{ => default_endpoints}/test_ogc.py (96%) rename tests/{ => default_endpoints}/test_ogc_features_manual.py (100%) rename tests/{ => default_endpoints}/test_parse_datetimes.py (100%) rename tests/{ => default_endpoints}/test_property_selection_shacl.py (100%) rename tests/{ => default_endpoints}/test_query_construction.py (100%) rename tests/{ => default_endpoints}/test_redirect_endpoint.py (100%) rename tests/{ => default_endpoints}/test_remote_prefixes.py (100%) rename tests/{ => default_endpoints}/test_search.py (100%) rename tests/{ => default_endpoints}/test_sparql.py (100%) diff --git a/prez/app.py b/prez/app.py index b36321aa..484d5474 100755 --- a/prez/app.py +++ b/prez/app.py @@ -29,10 +29,11 @@ PrefixNotFoundException, ) from prez.repositories import RemoteSparqlRepo, PyoxigraphRepo, OxrdflibRepo +from prez.routers.custom_endpoints import create_dynamic_router from prez.routers.identifier import router as identifier_router from prez.routers.management import router as management_router -from prez.routers.ogc_router import router as ogc_records_router from prez.routers.ogc_features_router import features_subapi +from prez.routers.base_router import router as base_prez_router from prez.routers.sparql import router as sparql_router from prez.services.app_service import ( healthcheck_sparql_endpoints, @@ -113,7 +114,7 @@ async def lifespan(app: FastAPI): await prefix_initialisation(app.state.repo) await retrieve_remote_template_queries(app.state.repo) await create_profiles_graph(app.state.repo) - await create_endpoints_graph(app.state.repo) + await create_endpoints_graph(app.state) await count_objects(app.state.repo) await populate_api_info() @@ -175,7 +176,6 @@ def assemble_app( app.state.settings = _settings app.include_router(management_router) - app.include_router(ogc_records_router) if _settings.enable_sparql_endpoint: app.include_router(sparql_router) if _settings.enable_ogc_features: @@ -183,6 +183,11 @@ def assemble_app( "/catalogs/{catalogId}/collections/{recordsCollectionId}/features", features_subapi, ) + if _settings.custom_endpoints: + app.include_router( + create_dynamic_router() + ) + app.include_router(base_prez_router) app.include_router(identifier_router) app.openapi = partial( prez_open_api_metadata, diff --git a/prez/config.py b/prez/config.py index ce5d6494..fbea1c26 100755 --- a/prez/config.py +++ b/prez/config.py @@ -73,13 +73,14 @@ class Settings(BaseSettings): DCTERMS.title, ] local_rdf_dir: str = "rdf" - endpoint_structure: Optional[Tuple[str, ...]] = ("catalogs", "collections", "items") + endpoint_structure: Optional[Tuple[str, ...]] = ("levelone", "leveltwo", "levelthree", "levelfour") system_endpoints: Optional[List[URIRef]] = [ EP["system/profile-listing"], EP["system/profile-object"], ] enable_sparql_endpoint: bool = False enable_ogc_features: bool = True + custom_endpoints: bool = False temporal_predicate: Optional[URIRef] = SDO.temporal endpoint_to_template_query_filename: Optional[Dict[str, str]] = {} diff --git a/prez/reference_data/endpoints/endpoint_metadata.ttl b/prez/reference_data/endpoints/base/endpoint_metadata.ttl similarity index 59% rename from prez/reference_data/endpoints/endpoint_metadata.ttl rename to prez/reference_data/endpoints/base/endpoint_metadata.ttl index 6ae6d6dc..7f176a16 100644 --- a/prez/reference_data/endpoints/endpoint_metadata.ttl +++ b/prez/reference_data/endpoints/base/endpoint_metadata.ttl @@ -29,36 +29,6 @@ ogce:cql-post ont:relevantShapes ex:CQL ; . -ogce:catalog-listing - a ont:ListingEndpoint ; - ont:relevantShapes ex:Catalogs ; -. - -ogce:catalog-object - a ont:ObjectEndpoint ; - ont:relevantShapes ex:Catalogs ; -. - -ogce:collection-listing - a ont:ListingEndpoint ; - ont:relevantShapes ex:Collections ; -. - -ogce:collection-object - a ont:ObjectEndpoint ; - ont:relevantShapes ex:Collections ; -. - -ogce:item-listing - a ont:ListingEndpoint ; - ont:relevantShapes ex:Feature , ex:ConceptSchemeConcept , ex:CollectionConcept , ex:Resource ; -. - -ogce:item-object - a ont:ObjectEndpoint ; - ont:relevantShapes ex:Feature , ex:ConceptSchemeConcept , ex:CollectionConcept , ex:Resource ; -. - ogce:search a ont:ListingEndpoint ; ont:relevantShapes ex:Search ; diff --git a/prez/reference_data/endpoints/endpoint_nodeshapes.ttl b/prez/reference_data/endpoints/base/endpoint_nodeshapes.ttl similarity index 50% rename from prez/reference_data/endpoints/endpoint_nodeshapes.ttl rename to prez/reference_data/endpoints/base/endpoint_nodeshapes.ttl index 33afc14d..c4b4bac9 100644 --- a/prez/reference_data/endpoints/endpoint_nodeshapes.ttl +++ b/prez/reference_data/endpoints/base/endpoint_nodeshapes.ttl @@ -12,65 +12,6 @@ @prefix skos: . @prefix altr-ext: . -ex:Catalogs - a sh:NodeShape ; - ont:hierarchyLevel 1 ; - sh:targetClass dcat:Catalog ; - sh:property [ - sh:path dcterms:hasPart ; - sh:or ( - [ sh:class skos:ConceptScheme ] - [ sh:class skos:Collection ] - [ sh:class dcat:Dataset ] - ) ; - ] . - -ex:Collections - a sh:NodeShape ; - ont:hierarchyLevel 2 ; - sh:targetClass skos:ConceptScheme , skos:Collection , dcat:Dataset ; - sh:property [ - sh:path [ sh:inversePath dcterms:hasPart ] ; - sh:class dcat:Catalog ; - ] . - - -ex:ConceptSchemeConcept - a sh:NodeShape ; - ont:hierarchyLevel 3 ; - sh:targetClass skos:Concept ; - sh:property [ - sh:path skos:inScheme ; - sh:class skos:ConceptScheme ; - ] , [ - sh:path ( skos:inScheme [ sh:inversePath dcterms:hasPart ] ); - sh:class dcat:Catalog ; - ] . - -ex:CollectionConcept - a sh:NodeShape ; - ont:hierarchyLevel 3 ; - sh:targetClass skos:Concept ; - sh:property [ - sh:path [ sh:inversePath skos:member ] ; - sh:class skos:Collection ; - ] , [ - sh:path ( [ sh:inversePath skos:member ] [ sh:inversePath dcterms:hasPart ] ); - sh:class dcat:Catalog ; - ] . - -ex:Resource - a sh:NodeShape ; - ont:hierarchyLevel 3 ; - sh:targetClass geo:FeatureCollection ; - sh:property [ - sh:path [ sh:inversePath dcterms:hasPart ] ; - sh:class dcat:Dataset ; - ] , [ - sh:path ( [ sh:inversePath dcterms:hasPart ] [ sh:inversePath dcterms:hasPart ] ); - sh:class dcat:Catalog ; - ] . - ex:Profiles a sh:NodeShape ; ont:hierarchyLevel 1 ; diff --git a/prez/reference_data/endpoints/features_metadata.ttl b/prez/reference_data/endpoints/features/features_metadata.ttl similarity index 100% rename from prez/reference_data/endpoints/features_metadata.ttl rename to prez/reference_data/endpoints/features/features_metadata.ttl diff --git a/prez/reference_data/endpoints/features_nodeshapes.ttl b/prez/reference_data/endpoints/features/features_nodeshapes.ttl similarity index 100% rename from prez/reference_data/endpoints/features_nodeshapes.ttl rename to prez/reference_data/endpoints/features/features_nodeshapes.ttl diff --git a/prez/routers/ogc_router.py b/prez/routers/base_router.py similarity index 78% rename from prez/routers/ogc_router.py rename to prez/routers/base_router.py index 48dade3f..9c93b37a 100755 --- a/prez/routers/ogc_router.py +++ b/prez/routers/base_router.py @@ -41,26 +41,6 @@ @router.get( path="/cql", summary="CQL GET endpoint", name=OGCE["cql-get"], responses=responses ) -@router.get( - "/catalogs", - summary="Catalog Listing", - name=OGCE["catalog-listing"], - responses=responses, -) -@router.get( - "/catalogs/{catalogId}/collections", - summary="Collection Listing", - name=OGCE["collection-listing"], - openapi_extra=ogc_extended_openapi_extras.get("collection-listing"), - responses=responses, -) -@router.get( - "/catalogs/{catalogId}/collections/{recordsCollectionId}/items", - summary="Item Listing", - name=OGCE["item-listing"], - openapi_extra=ogc_extended_openapi_extras.get("item-listing"), - responses=responses, -) @router.get( "/concept-hierarchy/{parent_curie}/top-concepts", summary="Top Concepts", @@ -144,9 +124,6 @@ async def cql_post_listings( # 1: /object?uri= # 2: /profiles/{profile_curie} -# 3: /catalogs/{catalogId} -# 4: /catalogs/{catalogId}/collections/{recordsCollectionId} -# 5: /catalogs/{catalogId}/collections/{recordsCollectionId}/items/{itemId} ######################################################################################################################## @@ -160,27 +137,6 @@ async def cql_post_listings( openapi_extra=ogc_extended_openapi_extras.get("profile-object"), responses=responses, ) -@router.get( - path="/catalogs/{catalogId}", - summary="Catalog Object", - name=OGCE["catalog-object"], - openapi_extra=ogc_extended_openapi_extras.get("catalog-object"), - responses=responses, -) -@router.get( - path="/catalogs/{catalogId}/collections/{recordsCollectionId}", - summary="Collection Object", - name=OGCE["collection-object"], - openapi_extra=ogc_extended_openapi_extras.get("collection-object"), - responses=responses, -) -@router.get( - path="/catalogs/{catalogId}/collections/{recordsCollectionId}/items/{itemId}", - summary="Item Object", - name=OGCE["item-object"], - openapi_extra=ogc_extended_openapi_extras.get("item-object"), - responses=responses, -) async def objects( pmts: NegotiatedPMTs = Depends(get_negotiated_pmts), endpoint_structure: tuple[str, ...] = Depends(get_endpoint_structure), diff --git a/prez/routers/custom_endpoints.py b/prez/routers/custom_endpoints.py new file mode 100644 index 00000000..83497703 --- /dev/null +++ b/prez/routers/custom_endpoints.py @@ -0,0 +1,163 @@ +import logging +from pathlib import Path as PLPath +from typing import List + +from fastapi import APIRouter, Depends +from fastapi import Path +from rdflib import Graph, RDF, RDFS +from sparql_grammar_pydantic import ConstructQuery + +from prez.dependencies import ( + get_data_repo, + get_system_repo, + generate_search_query, + cql_get_parser_dependency, + get_endpoint_nodeshapes, + get_negotiated_pmts, + get_profile_nodeshape, + get_endpoint_structure, + generate_concept_hierarchy_query, +) +from prez.models.query_params import QueryParams +from prez.reference_data.prez_ns import ONT +from prez.repositories import Repo +from prez.services.connegp_service import NegotiatedPMTs +from prez.services.listings import listing_function +from prez.services.objects import object_function +from prez.services.query_generation.concept_hierarchy import ConceptHierarchyQuery +from prez.services.query_generation.cql import CQLParser +from prez.services.query_generation.shacl import NodeShape + +log = logging.getLogger(__name__) + + +def load_routes() -> Graph: + g = Graph() + for file in (PLPath(__file__).parent.parent / "reference_data" / "endpoints" / "data_endpoints_custom").glob("*ttl"): + g.parse(file, format="ttl") + return g + + +def create_path_param(name: str, description: str, example: str): + return Path(..., description=description, example=example) + + +# Dynamic route handler +def create_dynamic_route_handler(route_type: str): + if route_type == "ListingEndpoint": + async def dynamic_list_handler( + query_params: QueryParams = Depends(), + endpoint_nodeshape: NodeShape = Depends(get_endpoint_nodeshapes), + pmts: NegotiatedPMTs = Depends(get_negotiated_pmts), + endpoint_structure: tuple[str, ...] = Depends(get_endpoint_structure), + profile_nodeshape: NodeShape = Depends(get_profile_nodeshape), + cql_parser: CQLParser = Depends(cql_get_parser_dependency), + search_query: ConstructQuery = Depends(generate_search_query), + concept_hierarchy_query: ConceptHierarchyQuery = Depends( + generate_concept_hierarchy_query + ), + data_repo: Repo = Depends(get_data_repo), + system_repo: Repo = Depends(get_system_repo), + ): + return await listing_function( + data_repo=data_repo, + system_repo=system_repo, + endpoint_nodeshape=endpoint_nodeshape, + endpoint_structure=endpoint_structure, + search_query=search_query, + concept_hierarchy_query=concept_hierarchy_query, + cql_parser=cql_parser, + pmts=pmts, + profile_nodeshape=profile_nodeshape, + query_params=query_params, + original_endpoint_type=ONT["ListingEndpoint"], + ) + return dynamic_list_handler + elif route_type == "ObjectEndpoint": + async def dynamic_object_handler( + pmts: NegotiatedPMTs = Depends(get_negotiated_pmts), + endpoint_structure: tuple[str, ...] = Depends(get_endpoint_structure), + profile_nodeshape: NodeShape = Depends(get_profile_nodeshape), + data_repo: Repo = Depends(get_data_repo), + system_repo: Repo = Depends(get_system_repo), + ): + return await object_function( + data_repo=data_repo, + system_repo=system_repo, + endpoint_structure=endpoint_structure, + pmts=pmts, + profile_nodeshape=profile_nodeshape, + ) + return dynamic_object_handler + + +# Extract path parameters from the path +def extract_path_params(path: str) -> List[str]: + return [part[1:-1] for part in path.split("/") if part.startswith("{") and part.endswith("}")] + + +# Add routes dynamically to the router +def add_routes(router: APIRouter): + g = load_routes() + routes = [] + for s in g.subjects(predicate=RDF.type, object=ONT.ListingEndpoint): + route = { + "path": str(g.value(s, ONT.apiPath)), + "name": str(s), + "description": str(g.value(s, RDFS.label)), + "type": "ListingEndpoint", + } + routes.append(route) + for s in g.subjects(predicate=RDF.type, object=ONT.ObjectEndpoint): + route = { + "path": str(g.value(s, ONT.apiPath)), + "name": str(s), + "description": str(g.value(s, RDFS.label)), + "type": "ObjectEndpoint", + } + routes.append(route) + + for route in routes: + path_param_names = extract_path_params(route["path"]) + + # Create path parameters using FastAPI's Path + path_params = { + param: create_path_param(param, f"Path parameter: {param}", f"example_{param}") + for param in path_param_names + } + + # Create OpenAPI extras for path parameters + openapi_extras = { + "parameters": [ + { + "in": "path", + "name": name, + "required": True, + "schema": {"type": "string", "example": param.example}, + "description": param.description, + } + for name, param in path_params.items() + ] + } + + # Create the endpoint function + endpoint = create_dynamic_route_handler(route["type"]) + + # Add the route to the router with OpenAPI extras + router.add_api_route( + name=route["name"], + path=route["path"], + endpoint=endpoint, + methods=["GET"], + description=route["description"], + openapi_extra=openapi_extras + ) + + log.info(f"Added route: {route['path']} with path parameters: {path_param_names}") + + +def create_dynamic_router() -> APIRouter: + log.info("Adding Custom Endpoints") + router = APIRouter(tags=["Custom Endpoints"]) + add_routes(router) + return router diff --git a/prez/services/app_service.py b/prez/services/app_service.py index ffbf8ff8..9a51790e 100755 --- a/prez/services/app_service.py +++ b/prez/services/app_service.py @@ -171,10 +171,26 @@ async def _add_prefixes_from_graph(g): return i -async def create_endpoints_graph(repo) -> Graph: - for f in (Path(__file__).parent.parent / "reference_data/endpoints").glob("*.ttl"): +async def create_endpoints_graph(app_state) -> Graph: + endpoints_root = Path(__file__).parent.parent / "reference_data/endpoints" + # OGC Features endpoints + if app_state.settings.enable_ogc_features: + for f in (endpoints_root / "features").glob("*.ttl"): + endpoints_graph_cache.parse(f) + # Custom data endpoints + if app_state.settings.custom_endpoints: + for f in (endpoints_root / "data_endpoints_custom").glob("*.ttl"): + endpoints_graph_cache.parse(f) + log.info("Custom endpoints loaded") + # Default data endpoints + else: + for f in (endpoints_root / "data_endpoints_default").glob("*.ttl"): + endpoints_graph_cache.parse(f) + await get_remote_endpoint_definitions(app_state.repo) + # Base endpoints + for f in (endpoints_root / "base").glob("*.ttl"): endpoints_graph_cache.parse(f) - await get_remote_endpoint_definitions(repo) + async def get_remote_endpoint_definitions(repo): diff --git a/prez/services/connegp_service.py b/prez/services/connegp_service.py index d4ff47f1..e15b12de 100755 --- a/prez/services/connegp_service.py +++ b/prez/services/connegp_service.py @@ -2,15 +2,13 @@ import re from enum import Enum from textwrap import dedent -from typing import List, Dict -from urllib.parse import urlencode from pydantic import BaseModel -from rdflib import Graph, Namespace, URIRef +from rdflib import Graph, Namespace, URIRef, SH +from prez.cache import endpoints_graph_cache from prez.config import settings from prez.exceptions.model_exceptions import NoProfilesException -from prez.models.ogc_features import Link from prez.repositories.base import Repo from prez.services.curie_functions import get_curie_id_for_uri, get_uri_for_curie_id @@ -209,7 +207,10 @@ async def _get_available(self) -> list[dict]: for result in repo_response[1][0][1] ] if not available: - raise NoProfilesException(self.classes) + if self.listing: + return [{"profile": URIRef("https://w3id.org/profile/mem"), "title": "Members", "mediatype": "text/anot+turtle", "class": "http://www.w3.org/2000/01/rdf-schema#Resource"}] + else: + return [{"profile": URIRef("https://prez.dev/profile/open-object"), "title": "Open profile", "mediatype": "text/anot+turtle", "class": "http://www.w3.org/2000/01/rdf-schema#Resource"}] return available def generate_response_headers(self) -> dict: @@ -237,6 +238,7 @@ def generate_response_headers(self) -> dict: def _compose_select_query(self) -> str: prez = Namespace("https://prez.dev/") profile_class = prez.ListingProfile if self.listing else prez.ObjectProfile + query_klasses = set(endpoints_graph_cache.objects(subject=None, predicate=SH.targetClass)) if self.requested_profiles: requested_profile = self.requested_profiles[0][0] else: @@ -261,10 +263,9 @@ def _compose_select_query(self) -> str: VALUES ?class {{{" ".join('<' + str(klass) + '>' for klass in self.classes)}}} ?class rdfs:subClassOf* ?mid . ?mid rdfs:subClassOf* ?base_class . - VALUES ?base_class {{ dcat:Dataset geo:FeatureCollection geo:Feature - skos:ConceptScheme skos:Concept skos:Collection - dcat:Catalog rdf:Resource dcat:Resource prof:Profile prez:SPARQLQuery - prez:SearchResult prez:CQLObjectList prez:Queryable prez:Object rdfs:Resource }} + VALUES ?base_class {{ {" ".join(klass.n3() for klass in query_klasses)} + prof:Profile prez:SPARQLQuery + prez:SearchResult prez:CQLObjectList prez:Object rdfs:Resource }} ?profile altr-ext:constrainsClass ?class ; altr-ext:hasResourceFormat ?format ; dcterms:title ?title .\ @@ -306,10 +307,7 @@ def _generate_mediatype_if_statements(self) -> str: async def _do_query(self, query: str) -> tuple[Graph, list]: response = await self.system_repo.send_queries([], [(None, query)]) - if not response[1][0][1]: - raise NoProfilesException(self.classes) - - if settings.log_level == "DEBUG": + if response[1][0][1] and settings.log_level == "DEBUG": from tabulate import tabulate table_data = [ diff --git a/prez/services/generate_endpoint_rdf.py b/prez/services/generate_endpoint_rdf.py new file mode 100644 index 00000000..5c3ef7ac --- /dev/null +++ b/prez/services/generate_endpoint_rdf.py @@ -0,0 +1,178 @@ +import json +from pathlib import Path + +from rdflib import Graph, Namespace, RDF, Literal, RDFS, SH, URIRef, BNode +from rdflib.collection import Collection + +from prez.reference_data.prez_ns import ONT + + + +EX = Namespace("http://example.org/") +TEMP = Namespace("http://temporary/") + + +def add_endpoint(g, endpoint_type, name, api_path, shapes_name, i): + hl = (i + 2) // 2 + endpoint_uri = EX[f"{name}-{endpoint_type}"] + title_cased = f"{name.title()} {endpoint_type.title()}" + + g.add((endpoint_uri, RDF.type, ONT[f"{endpoint_type.title()}Endpoint"])) + g.add((endpoint_uri, RDFS.label, Literal(title_cased))) + g.add((endpoint_uri, ONT.apiPath, Literal(api_path))) + g.add((endpoint_uri, TEMP.relevantShapes, Literal(hl))) + + +def create_endpoint_metadata(data, g): + for route in data['routes']: + fullApiPath = route["fullApiPath"] + components = fullApiPath.split("/")[1:] + + for i in range(len(components)): + name = components[i] if i % 2 == 0 else components[i - 1] + api_path = f"/{'/'.join(components[:i + 1])}" + shapes_name = name.title() + + if i % 2 == 0: + add_endpoint(g, "listing", name, api_path, shapes_name, i) + else: + add_endpoint(g, "object", name, api_path, shapes_name, i) + +def process_relations(data): + for route in data['routes']: + levels = {} + for hier_rel in route["hierarchiesRelations"]: + hierarchy_dict = {h["hierarchyLevel"]: h for h in hier_rel["hierarchy"]} + for relation in hier_rel["relations"]: + rel_key = tuple(sorted(relation.items())) # Sort items before creating tuple + level_from = relation["levelFrom"] + level_to = relation["levelTo"] + klass_from = hierarchy_dict[level_from] + klass_to = hierarchy_dict[level_to] + klass_from_key = tuple(sorted(klass_from.items())) # Sort items before creating tuple + klass_to_key = tuple(sorted(klass_to.items())) # Sort items before creating tuple + + if rel_key in levels: + levels[rel_key]["klasses_from"].add(klass_from_key) + levels[rel_key]["klasses_to"].add(klass_to_key) + else: + levels[rel_key] = { + "klasses_from": {klass_from_key}, + "klasses_to": {klass_to_key} + } + return levels + + +def process_levels(levels: dict, g: Graph): + unique_suffix = 1 + shape_names = set() + for i, (k, v) in enumerate(levels.items()): + proposed_shape_uri = EX[f"shape-{k[2][1]}"] + if proposed_shape_uri not in shape_names: + shape_names.add(proposed_shape_uri) + shape_uri = proposed_shape_uri + else: + shape_uri = EX[f"shape-{k[2][1]}-{unique_suffix}"] + unique_suffix += 1 + g.add((shape_uri, RDF.type, SH.NodeShape)) + g.add((shape_uri, ONT.hierarchyLevel, Literal(k[2][1]))) # hierarchyLevel = levelTo + klasses_to = [] + klasses_from = [] + for tup in v["klasses_to"]: + klasses_to.append(URIRef(tup[2][1])) + for klass in klasses_to: + g.add((shape_uri, SH.targetClass, klass)) + for tup in v["klasses_from"]: + klasses_from.append(URIRef(tup[2][1])) + prop_bn = BNode() + g.add((shape_uri, SH.property, prop_bn)) + if k[1][1] > k[2][1]: # levelFrom > levelTo - top level endpoint only + klass_bns = [] + or_bn = BNode() + g.add((prop_bn, SH["or"], or_bn)) + for klass in klasses_from: + klass_bn = BNode() + g.add((klass_bn, SH["class"], klass)) + klass_bns.append(klass_bn) + Collection(g, or_bn, klass_bns) + g.add((prop_bn, SH["path"], URIRef(k[3][1]))) + elif k[1][1] < k[2][1]: # levelFrom < levelTo + if k[0][1] == "outbound": + path_bn = BNode() + g.add((prop_bn, SH.path, path_bn)) + g.add((path_bn, SH.inversePath, URIRef(k[3][1]))) # relation + else: + g.add((prop_bn, SH.path, URIRef(k[3][1]))) + g.add((prop_bn, SH["class"], klasses_from[0])) # klass_from + for tup in v["klasses_to"]: + if tup[1][1] > 2: # hierarchy level > 2 + klass_to_match = klasses_from[0] + for rel, klass_info in levels.items(): + for n in klass_info["klasses_to"]: + if klass_to_match == URIRef(n[2][1]): + second_rel = rel + second_klass = klass_info + break + second_prop_bn = BNode() + second_path_bn = BNode() + g.add((shape_uri, SH.property, second_prop_bn)) + g.add((second_prop_bn, SH.path, second_path_bn)) + list_comps = [] + if k[0][1] == "outbound": + inverse_bn = BNode() + g.add((inverse_bn, SH.inversePath, URIRef(k[3][1]))) # relation + list_comps.append(inverse_bn) + else: + list_comps.append(URIRef(k[3][1])) + if second_rel[0][1] == "outbound": + inverse_bn = BNode() + g.add((inverse_bn, SH.inversePath, URIRef(second_rel[3][1]))) # relation + list_comps.append(inverse_bn) + else: + list_comps.append(URIRef(second_rel[3][1])) + Collection(g, second_path_bn, list_comps) + for tup in second_klass["klasses_from"]: + g.add((second_prop_bn, SH["class"], URIRef(tup[2][1]))) + + + +def read_json(file_path): + with open(file_path, "r") as f: + data = json.load(f) + for route in data['routes']: + for hr in route["hierarchiesRelations"]: + for relation in hr["relations"]: + if relation["levelFrom"] == 1 and relation["levelTo"] == 2: + inverted_relation = { + "levelFrom": 2, + "levelTo": 1, + "rdfPredicate": relation["rdfPredicate"], + "direction": "inbound" if relation["direction"] == "outbound" else "outbound" + } + hr["relations"].append(inverted_relation) + return data + + +def link_endpoints_shapes(endpoints_g, shapes_g, links_g): + for s, p, o in shapes_g.triples((None, ONT.hierarchyLevel, None)): + for s2, p2, o2 in endpoints_g.triples((None, TEMP.relevantShapes, None)): + if o == o2: + links_g.add((s2, ONT.relevantShapes, s)) + endpoints_g.remove((s2, TEMP.relevantShapes, o2)) + + +if __name__ == "__main__": + file_path = Path(__file__).parent.parent.parent / "test_data" / "custom_endpoints.json" + data = read_json(file_path) + g = Graph() + create_endpoint_metadata(data, g) + levels = process_relations(data) + g2 = Graph() + results = process_levels(levels, g2) + g3 = Graph() + link_endpoints_shapes(g, g2, g3) + complete = g + g2 + g3 + complete.bind("ont", ONT) + complete.bind("ex", EX) + file_path = Path(__file__).parent.parent / "reference_data" / "endpoints" / "data_endpoints_custom" / "custom_endpoints.ttl" + complete.serialize(destination=file_path, format="turtle") diff --git a/test_data/custom_endpoints.json b/test_data/custom_endpoints.json new file mode 100644 index 00000000..0b23e02e --- /dev/null +++ b/test_data/custom_endpoints.json @@ -0,0 +1,55 @@ +{ + "routes": [ + { + "name": "TestDefault", + "fullApiPath": "/levelone/{level1Id}/leveltwo/{level2Id}/levelthree/{level3Id}/levelfour/{level4Id}", + "hierarchiesRelations": [ + { + "name": "TestHierarchy", + "hierarchy": [ + { + "rdfClass": "http://example.com/Level1", + "className": "Level1", + "hierarchyLevel": 1 + }, + { + "rdfClass": "http://example.com/Level2", + "className": "Level2", + "hierarchyLevel": 2 + }, + { + "rdfClass": "http://example.com/Level3", + "className": "Level3", + "hierarchyLevel": 3 + }, + { + "rdfClass": "http://example.com/Level4", + "className": "Level4", + "hierarchyLevel": 4 + } + ], + "relations": [ + { + "levelFrom": 1, + "levelTo": 2, + "direction": "outbound", + "rdfPredicate": "http://example.com/l1tol2hasChild" + }, + { + "levelFrom": 2, + "levelTo": 3, + "direction": "inbound", + "rdfPredicate": "http://example.com/l3tol2hasParent" + }, + { + "levelFrom": 3, + "levelTo": 4, + "direction": "outbound", + "rdfPredicate": "http://example.com/l3tol4hasChild" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/custom_endpoints/conftest.py b/tests/custom_endpoints/conftest.py new file mode 100755 index 00000000..46a2576b --- /dev/null +++ b/tests/custom_endpoints/conftest.py @@ -0,0 +1,118 @@ +import os + +from rdflib import Graph, URIRef, RDF +from rdflib.namespace import GEO +from starlette.routing import Mount + +# comment / uncomment for the CQL tests - cannot figure out how to get a different conftest picked up. +os.environ["SPARQL_REPO_TYPE"] = "pyoxigraph" + +# os.environ["SPARQL_ENDPOINT"] = "http://localhost:3030/dataset" +# os.environ["SPARQL_REPO_TYPE"] = "remote" +os.environ["ENABLE_SPARQL_ENDPOINT"] = "true" +os.environ["CUSTOM_ENDPOINTS"] = "true" + +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient +from pyoxigraph.pyoxigraph import Store + +from prez.app import assemble_app +from prez.dependencies import get_data_repo +from prez.repositories import Repo, PyoxigraphRepo + + +@pytest.fixture(scope="module") +def test_store() -> Store: + # Create a new pyoxigraph Store + store = Store() + + for file in Path(__file__).parent.parent.glob("../test_data/*.ttl"): + store.load(file.read_bytes(), "text/turtle") + + return store + + +@pytest.fixture(scope="module") +def test_repo(test_store: Store) -> Repo: + # Create a PyoxigraphQuerySender using the test_store + return PyoxigraphRepo(test_store) + + +@pytest.fixture(scope="module") +def client(test_repo: Repo) -> TestClient: + # Override the dependency to use the test_repo + def override_get_repo(): + return test_repo + + app = assemble_app() + + app.dependency_overrides[get_data_repo] = override_get_repo + + for route in app.routes: + if isinstance(route, Mount): + route.app.dependency_overrides[get_data_repo] = override_get_repo + + with TestClient(app) as c: + yield c + + # Remove the override to ensure subsequent tests are unaffected + app.dependency_overrides.clear() + + +@pytest.fixture(scope="module") +def client_no_override() -> TestClient: + + app = assemble_app() + + with TestClient(app) as c: + yield c + + +@pytest.fixture() +def a_spaceprez_catalog_link(client): + r = client.get("/catalogs") + g = Graph().parse(data=r.text) + cat_uri = URIRef("https://example.com/spaceprez/SpacePrezCatalog") + link = g.value(cat_uri, URIRef(f"https://prez.dev/link", None)) + return link + + +@pytest.fixture() +def a_spaceprez_dataset_link(client, a_spaceprez_catalog_link): + r = client.get(f"{a_spaceprez_catalog_link}/collections") + g = Graph().parse(data=r.text) + ds_uri = URIRef("https://example.com/spaceprez/SpacePrezDataset") + link = g.value(ds_uri, URIRef(f"https://prez.dev/link", None)) + return link + + +@pytest.fixture() +def an_fc_link(client, a_spaceprez_dataset_link): + return f"{a_spaceprez_dataset_link}/features/collections/spcprz:FeatureCollection" + + +@pytest.fixture() +def a_feature_link(client, an_fc_link): + return f"{an_fc_link}/items/spcprz:Feature1" + + +@pytest.fixture() +def a_catprez_catalog_link(client): + # get link for first catalog + r = client.get("/catalogs") + g = Graph().parse(data=r.text) + member_uri = URIRef("https://example.com/CatalogOne") + link = g.value(member_uri, URIRef(f"https://prez.dev/link", None)) + return link + + +@pytest.fixture() +def a_resource_link(client, a_catprez_catalog_link): + r = client.get(a_catprez_catalog_link) + g = Graph().parse(data=r.text) + links = g.objects(subject=None, predicate=URIRef(f"https://prez.dev/link")) + for link in links: + if link != a_catprez_catalog_link: + return link diff --git a/tests/TO_FIX_test_dd_profiles.py b/tests/default_endpoints/TO_FIX_test_dd_profiles.py similarity index 100% rename from tests/TO_FIX_test_dd_profiles.py rename to tests/default_endpoints/TO_FIX_test_dd_profiles.py diff --git a/tests/TO_FIX_test_endpoints_vocprez.py b/tests/default_endpoints/TO_FIX_test_endpoints_vocprez.py similarity index 100% rename from tests/TO_FIX_test_endpoints_vocprez.py rename to tests/default_endpoints/TO_FIX_test_endpoints_vocprez.py diff --git a/tests/TO_FIX_test_search.py b/tests/default_endpoints/TO_FIX_test_search.py similarity index 100% rename from tests/TO_FIX_test_search.py rename to tests/default_endpoints/TO_FIX_test_search.py diff --git a/tests/default_endpoints/__init__.py b/tests/default_endpoints/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/_test_count.py b/tests/default_endpoints/_test_count.py similarity index 100% rename from tests/_test_count.py rename to tests/default_endpoints/_test_count.py diff --git a/tests/_test_cql.py b/tests/default_endpoints/_test_cql.py similarity index 100% rename from tests/_test_cql.py rename to tests/default_endpoints/_test_cql.py diff --git a/tests/_test_cql_fuseki.py b/tests/default_endpoints/_test_cql_fuseki.py similarity index 100% rename from tests/_test_cql_fuseki.py rename to tests/default_endpoints/_test_cql_fuseki.py diff --git a/tests/_test_curie_generation.py b/tests/default_endpoints/_test_curie_generation.py similarity index 100% rename from tests/_test_curie_generation.py rename to tests/default_endpoints/_test_curie_generation.py diff --git a/tests/conftest.py b/tests/default_endpoints/conftest.py similarity index 97% rename from tests/conftest.py rename to tests/default_endpoints/conftest.py index 17ffa5ec..f183164f 100755 --- a/tests/conftest.py +++ b/tests/default_endpoints/conftest.py @@ -27,7 +27,7 @@ def test_store() -> Store: # Create a new pyoxigraph Store store = Store() - for file in Path(__file__).parent.glob("../test_data/*.ttl"): + for file in (Path(__file__).parent.parent.parent / "test_data").glob("*.ttl"): store.load(file.read_bytes(), "text/turtle") return store diff --git a/tests/cql-fuseki-config.ttl b/tests/default_endpoints/cql-fuseki-config.ttl similarity index 100% rename from tests/cql-fuseki-config.ttl rename to tests/default_endpoints/cql-fuseki-config.ttl diff --git a/tests/test_alt_profiles.py b/tests/default_endpoints/test_alt_profiles.py similarity index 100% rename from tests/test_alt_profiles.py rename to tests/default_endpoints/test_alt_profiles.py diff --git a/tests/test_bnode.py b/tests/default_endpoints/test_bnode.py similarity index 87% rename from tests/test_bnode.py rename to tests/default_endpoints/test_bnode.py index 8d480009..e5c2d807 100755 --- a/tests/test_bnode.py +++ b/tests/default_endpoints/test_bnode.py @@ -1,4 +1,4 @@ -import pathlib +from pathlib import Path import pytest from rdflib import Graph, URIRef @@ -6,7 +6,6 @@ from prez.bnode import get_bnode_depth -WORKING_DIR = pathlib.Path().parent @pytest.mark.parametrize( @@ -19,7 +18,7 @@ ], ) def test_bnode_depth(input_file: str, iri: str, expected_depth: int) -> None: - file = WORKING_DIR / "test_data" / input_file + file = Path(__file__).parent.parent.parent / "test_data" / input_file graph = Graph() graph.parse(file) diff --git a/tests/test_connegp.py b/tests/default_endpoints/test_connegp.py similarity index 100% rename from tests/test_connegp.py rename to tests/default_endpoints/test_connegp.py diff --git a/tests/test_cql_time.py b/tests/default_endpoints/test_cql_time.py similarity index 91% rename from tests/test_cql_time.py rename to tests/default_endpoints/test_cql_time.py index 4d40790f..40f0c821 100755 --- a/tests/test_cql_time.py +++ b/tests/default_endpoints/test_cql_time.py @@ -42,16 +42,16 @@ ) def test_time_funcs(cql_json_filename, output_query_filename): cql_json_path = ( - Path(__file__).parent.parent / f"test_data/cql/input/{cql_json_filename}" + Path(__file__).parent.parent.parent / f"test_data/cql/input/{cql_json_filename}" ) cql_json = json.loads(cql_json_path.read_text()) reference_query = ( - Path(__file__).parent.parent + Path(__file__).parent.parent.parent / f"test_data/cql/expected_generated_queries/{output_query_filename}" ).read_text() context = json.load( ( - Path(__file__).parent.parent + Path(__file__).parent.parent.parent / "prez/reference_data/cql/default_context.json" ).open() ) diff --git a/tests/test_curie_endpoint.py b/tests/default_endpoints/test_curie_endpoint.py similarity index 100% rename from tests/test_curie_endpoint.py rename to tests/default_endpoints/test_curie_endpoint.py diff --git a/tests/test_endpoints_cache.py b/tests/default_endpoints/test_endpoints_cache.py similarity index 100% rename from tests/test_endpoints_cache.py rename to tests/default_endpoints/test_endpoints_cache.py diff --git a/tests/test_endpoints_catprez.py b/tests/default_endpoints/test_endpoints_catprez.py similarity index 100% rename from tests/test_endpoints_catprez.py rename to tests/default_endpoints/test_endpoints_catprez.py diff --git a/tests/test_endpoints_concept_hierarchy.py b/tests/default_endpoints/test_endpoints_concept_hierarchy.py similarity index 100% rename from tests/test_endpoints_concept_hierarchy.py rename to tests/default_endpoints/test_endpoints_concept_hierarchy.py diff --git a/tests/test_endpoints_management.py b/tests/default_endpoints/test_endpoints_management.py similarity index 100% rename from tests/test_endpoints_management.py rename to tests/default_endpoints/test_endpoints_management.py diff --git a/tests/test_endpoints_object.py b/tests/default_endpoints/test_endpoints_object.py similarity index 100% rename from tests/test_endpoints_object.py rename to tests/default_endpoints/test_endpoints_object.py diff --git a/tests/test_endpoints_ok.py b/tests/default_endpoints/test_endpoints_ok.py similarity index 100% rename from tests/test_endpoints_ok.py rename to tests/default_endpoints/test_endpoints_ok.py diff --git a/tests/test_endpoints_profiles.py b/tests/default_endpoints/test_endpoints_profiles.py similarity index 100% rename from tests/test_endpoints_profiles.py rename to tests/default_endpoints/test_endpoints_profiles.py diff --git a/tests/test_endpoints_spaceprez.py b/tests/default_endpoints/test_endpoints_spaceprez.py similarity index 100% rename from tests/test_endpoints_spaceprez.py rename to tests/default_endpoints/test_endpoints_spaceprez.py diff --git a/tests/test_geojson_to_wkt.py b/tests/default_endpoints/test_geojson_to_wkt.py similarity index 100% rename from tests/test_geojson_to_wkt.py rename to tests/default_endpoints/test_geojson_to_wkt.py diff --git a/tests/test_node_selection_shacl.py b/tests/default_endpoints/test_node_selection_shacl.py similarity index 76% rename from tests/test_node_selection_shacl.py rename to tests/default_endpoints/test_node_selection_shacl.py index 8b2c70c0..9b895e30 100755 --- a/tests/test_node_selection_shacl.py +++ b/tests/default_endpoints/test_node_selection_shacl.py @@ -1,3 +1,5 @@ +from pathlib import Path + import pytest from rdflib import Graph, URIRef @@ -7,11 +9,13 @@ from sparql_grammar_pydantic import Var endpoints_graph = Graph().parse( - "prez/reference_data/endpoints/endpoint_nodeshapes.ttl", format="turtle" + Path(__file__).parent.parent.parent + / "prez/reference_data/endpoints/data_endpoints_default/default_endpoints.ttl", + format="turtle", ) -@pytest.mark.parametrize("nodeshape_uri", ["http://example.org/ns#Collections"]) +@pytest.mark.parametrize("nodeshape_uri", ["http://example.org/shape-2"]) def test_nodeshape_parsing(nodeshape_uri): ns = NodeShape( uri=URIRef(nodeshape_uri), @@ -20,8 +24,8 @@ def test_nodeshape_parsing(nodeshape_uri): focus_node=Var(value="focus_node"), ) assert ns.targetClasses == [ - URIRef("http://www.w3.org/2004/02/skos/core#ConceptScheme"), URIRef("http://www.w3.org/2004/02/skos/core#Collection"), + URIRef("http://www.w3.org/2004/02/skos/core#ConceptScheme"), URIRef("http://www.w3.org/ns/dcat#Dataset"), ] assert len(ns.propertyShapesURIs) == 1 @@ -29,7 +33,7 @@ def test_nodeshape_parsing(nodeshape_uri): @pytest.mark.parametrize( "nodeshape_uri", - ["http://example.org/ns#ConceptSchemeConcept"], + ["http://example.org/shape-3"], ) def test_nodeshape_to_grammar(nodeshape_uri): ns = NodeShape( diff --git a/tests/test_ogc.py b/tests/default_endpoints/test_ogc.py similarity index 96% rename from tests/test_ogc.py rename to tests/default_endpoints/test_ogc.py index 280ee4d9..043c98ff 100644 --- a/tests/test_ogc.py +++ b/tests/default_endpoints/test_ogc.py @@ -16,7 +16,7 @@ def test_store() -> Store: # Create a new pyoxigraph Store store = Store() - file = Path(__file__).parent.parent / "test_data/ogc_features.ttl" + file = Path(__file__).parent.parent.parent / "test_data/ogc_features.ttl" store.load(file.read_bytes(), "text/turtle") return store diff --git a/tests/test_ogc_features_manual.py b/tests/default_endpoints/test_ogc_features_manual.py similarity index 100% rename from tests/test_ogc_features_manual.py rename to tests/default_endpoints/test_ogc_features_manual.py diff --git a/tests/test_parse_datetimes.py b/tests/default_endpoints/test_parse_datetimes.py similarity index 100% rename from tests/test_parse_datetimes.py rename to tests/default_endpoints/test_parse_datetimes.py diff --git a/tests/test_property_selection_shacl.py b/tests/default_endpoints/test_property_selection_shacl.py similarity index 100% rename from tests/test_property_selection_shacl.py rename to tests/default_endpoints/test_property_selection_shacl.py diff --git a/tests/test_query_construction.py b/tests/default_endpoints/test_query_construction.py similarity index 100% rename from tests/test_query_construction.py rename to tests/default_endpoints/test_query_construction.py diff --git a/tests/test_redirect_endpoint.py b/tests/default_endpoints/test_redirect_endpoint.py similarity index 100% rename from tests/test_redirect_endpoint.py rename to tests/default_endpoints/test_redirect_endpoint.py diff --git a/tests/test_remote_prefixes.py b/tests/default_endpoints/test_remote_prefixes.py similarity index 100% rename from tests/test_remote_prefixes.py rename to tests/default_endpoints/test_remote_prefixes.py diff --git a/tests/test_search.py b/tests/default_endpoints/test_search.py similarity index 100% rename from tests/test_search.py rename to tests/default_endpoints/test_search.py diff --git a/tests/test_sparql.py b/tests/default_endpoints/test_sparql.py similarity index 100% rename from tests/test_sparql.py rename to tests/default_endpoints/test_sparql.py From 14d8056215aed9bc2e9dd84151e577925c82adb1 Mon Sep 17 00:00:00 2001 From: david Date: Wed, 2 Oct 2024 10:15:58 +1000 Subject: [PATCH 2/7] Add simple HTML page --- prez/app.py | 8 +- prez/config.py | 3 +- prez/models/endpoint_config.py | 96 ++++++++++++ prez/routers/management.py | 37 ++++- prez/services/generate_endpoint_rdf.py | 41 ++--- prez/static/endpoint_config.html | 198 +++++++++++++++++++++++++ 6 files changed, 355 insertions(+), 28 deletions(-) create mode 100644 prez/models/endpoint_config.py create mode 100644 prez/static/endpoint_config.html diff --git a/prez/app.py b/prez/app.py index 484d5474..feab4815 100755 --- a/prez/app.py +++ b/prez/app.py @@ -8,6 +8,7 @@ from fastapi.openapi.utils import get_openapi from rdflib import Graph from starlette.middleware.cors import CORSMiddleware +from starlette.staticfiles import StaticFiles from prez.config import settings, Settings from prez.dependencies import ( @@ -31,7 +32,7 @@ from prez.repositories import RemoteSparqlRepo, PyoxigraphRepo, OxrdflibRepo from prez.routers.custom_endpoints import create_dynamic_router from prez.routers.identifier import router as identifier_router -from prez.routers.management import router as management_router +from prez.routers.management import router as management_router, config_router from prez.routers.ogc_features_router import features_subapi from prez.routers.base_router import router as base_prez_router from prez.routers.sparql import router as sparql_router @@ -178,9 +179,11 @@ def assemble_app( app.include_router(management_router) if _settings.enable_sparql_endpoint: app.include_router(sparql_router) + if _settings.configuration_mode: + app.include_router(config_router) if _settings.enable_ogc_features: app.mount( - "/catalogs/{catalogId}/collections/{recordsCollectionId}/features", + "/catalogs/{catalogId}/datasets/{datasetId}/features", features_subapi, ) if _settings.custom_endpoints: @@ -189,6 +192,7 @@ def assemble_app( ) app.include_router(base_prez_router) app.include_router(identifier_router) + app.mount("/static", StaticFiles(directory="static"), name="static") app.openapi = partial( prez_open_api_metadata, title=title, diff --git a/prez/config.py b/prez/config.py index fbea1c26..f4a1c2d9 100755 --- a/prez/config.py +++ b/prez/config.py @@ -73,7 +73,7 @@ class Settings(BaseSettings): DCTERMS.title, ] local_rdf_dir: str = "rdf" - endpoint_structure: Optional[Tuple[str, ...]] = ("levelone", "leveltwo", "levelthree", "levelfour") + endpoint_structure: Optional[Tuple[str, ...]] = ("catalogs", "collections", "items") system_endpoints: Optional[List[URIRef]] = [ EP["system/profile-listing"], EP["system/profile-object"], @@ -81,6 +81,7 @@ class Settings(BaseSettings): enable_sparql_endpoint: bool = False enable_ogc_features: bool = True custom_endpoints: bool = False + configuration_mode: bool = False temporal_predicate: Optional[URIRef] = SDO.temporal endpoint_to_template_query_filename: Optional[Dict[str, str]] = {} diff --git a/prez/models/endpoint_config.py b/prez/models/endpoint_config.py new file mode 100644 index 00000000..c959aa32 --- /dev/null +++ b/prez/models/endpoint_config.py @@ -0,0 +1,96 @@ +from typing import List, Literal + +from pydantic import BaseModel, field_validator +from rdflib import URIRef + + +class HierarchyItem(BaseModel): + rdfClass: str + className: str + hierarchyLevel: int + + @classmethod + @field_validator("rdfClass") + def validate_uri(cls, v: str) -> str: + try: + URIRef(v) + return v + except ValueError as e: + raise ValueError(f"Invalid URI: {v}. Error: {str(e)}") + + +class Relation(BaseModel): + levelFrom: int + levelTo: int + direction: Literal["outbound", "inbound"] + rdfPredicate: str + + @classmethod + @field_validator("rdfPredicate") + def validate_uri(cls, v: str) -> str: + try: + URIRef(v) + return v + except ValueError as e: + raise ValueError(f"Invalid URI: {v}. Error: {str(e)}") + + +class HierarchyRelation(BaseModel): + name: str + hierarchy: List[HierarchyItem] + relations: List[Relation] + + @classmethod + @field_validator("relations") + def validate_relations_count(cls, v: List[Relation], values: dict) -> List[Relation]: + hierarchy = values.get('hierarchy') + if hierarchy and len(v) != len(hierarchy) - 1: + raise ValueError( + f"Number of relations ({len(v)}) should be one less than the number of hierarchy items ({len(hierarchy)})") + return v + + +class Route(BaseModel): + name: str + fullApiPath: str + hierarchiesRelations: List[HierarchyRelation] + + +class RootModel(BaseModel): + routes: List[Route] + + +configure_endpoings_example = { + "configName": "Prez Example Endpoint Configuration", + "routes": [ + { + "name": "Single hierarchy with Datasets within Catalogs", + "fullApiPath": "/catalogs/{catalogId}/datasets/{datasetId}", + "hierarchiesRelations": [ + { + "name": "Catalogue of Datasets", + "hierarchy": [ + { + "rdfClass": "http://www.w3.org/ns/dcat#Catalog", + "className": "Catalog", + "hierarchyLevel": 1 + }, + { + "rdfClass": "http://www.w3.org/ns/dcat#Dataset", + "className": "DCAT Dataset", + "hierarchyLevel": 2 + } + ], + "relations": [ + { + "levelFrom": 1, + "levelTo": 2, + "direction": "outbound", + "rdfPredicate": "http://purl.org/dc/terms/hasPart" + } + ] + } + ] + } + ] +} diff --git a/prez/routers/management.py b/prez/routers/management.py index 5ae2e21a..a5552581 100755 --- a/prez/routers/management.py +++ b/prez/routers/management.py @@ -5,23 +5,28 @@ from typing import Optional from aiocache import caches -from fastapi import APIRouter, Depends, Query +from fastapi import APIRouter, Depends, Query, HTTPException, Body +from pydantic import ValidationError from rdflib import BNode, VANN from rdflib import Graph, URIRef, Literal from rdflib.collection import Collection from starlette.requests import Request -from starlette.responses import PlainTextResponse, StreamingResponse +from starlette.responses import PlainTextResponse, StreamingResponse, Response from prez.cache import endpoints_graph_cache, prefix_graph from prez.config import settings from prez.dependencies import get_system_repo from prez.enums import JSONMediaType, NonAnnotatedRDFMediaType +from prez.models.endpoint_config import RootModel, configure_endpoings_example from prez.reference_data.prez_ns import PREZ from prez.renderers.renderer import return_rdf, return_from_graph from prez.repositories import Repo from prez.services.connegp_service import RDF_MEDIATYPES, NegotiatedPMTs +from prez.services.generate_endpoint_rdf import create_endpoint_rdf router = APIRouter(tags=["Management"]) +config_router = APIRouter(tags=["Configuration"]) + log = logging.getLogger(__name__) @@ -81,7 +86,7 @@ async def return_tbox_cache(request: Request): pred_obj = pickle.loads(pred_obj_bytes) for pred, obj in pred_obj: if ( - pred_obj + pred_obj ): # cache entry for a URI can be empty - i.e. no annotations found for URI # Add the expanded triple (subject, predicate, object) to 'annotations_g' cache_g.add((subject, pred, obj)) @@ -118,9 +123,9 @@ async def return_annotation_predicates(): @router.get("/prefixes", summary="Show prefixes known to prez") async def show_prefixes( - mediatype: Optional[NonAnnotatedRDFMediaType | JSONMediaType] = Query( - default=NonAnnotatedRDFMediaType.TURTLE, alias="_mediatype" - ) + mediatype: Optional[NonAnnotatedRDFMediaType | JSONMediaType] = Query( + default=NonAnnotatedRDFMediaType.TURTLE, alias="_mediatype" + ) ): """Returns the prefixes known to prez""" mediatype_str = str(mediatype.value) @@ -135,3 +140,23 @@ async def show_prefixes( g.add((bn, VANN.preferredNamespaceUri, Literal(namespace))) content = io.BytesIO(g.serialize(format=mediatype_str, encoding="utf-8")) return StreamingResponse(content=content, media_type=mediatype_str) + + + +@config_router.post("/configure-endpoints", summary="Configuration") +async def submit_config(config: RootModel = Body( + ..., + examples=[configure_endpoings_example] +) +): + try: + create_endpoint_rdf(config.model_dump()) + return {"message": f"Configuration received successfully. {len(config.routes)} routes processed."} + except ValidationError as e: + raise HTTPException(status_code=400, detail=str(e)) + + +@config_router.get("/configure-endpoints") +async def open_config_page(): + """Redirects to the endpoint configuration page""" + return Response(status_code=302, headers={"Location": "/static/endpoint_config.html"}) diff --git a/prez/services/generate_endpoint_rdf.py b/prez/services/generate_endpoint_rdf.py index 5c3ef7ac..fe737a03 100644 --- a/prez/services/generate_endpoint_rdf.py +++ b/prez/services/generate_endpoint_rdf.py @@ -136,21 +136,24 @@ def process_levels(levels: dict, g: Graph): -def read_json(file_path): - with open(file_path, "r") as f: - data = json.load(f) - for route in data['routes']: - for hr in route["hierarchiesRelations"]: - for relation in hr["relations"]: - if relation["levelFrom"] == 1 and relation["levelTo"] == 2: - inverted_relation = { - "levelFrom": 2, - "levelTo": 1, - "rdfPredicate": relation["rdfPredicate"], - "direction": "inbound" if relation["direction"] == "outbound" else "outbound" - } - hr["relations"].append(inverted_relation) - return data +def add_inverse_for_top_level(data): + """ + the RDF relation between the first and second endpoints is reused in reverse so endpoints can be defined based on + their relation to each other rather than needing to say put the class of objects at the top level in some arbitrary + collection + """ + for route in data['routes']: + for hr in route["hierarchiesRelations"]: + for relation in hr["relations"]: + if relation["levelFrom"] == 1 and relation["levelTo"] == 2: + inverted_relation = { + "levelFrom": 2, + "levelTo": 1, + "rdfPredicate": relation["rdfPredicate"], + "direction": "inbound" if relation["direction"] == "outbound" else "outbound" + } + hr["relations"].append(inverted_relation) + return data def link_endpoints_shapes(endpoints_g, shapes_g, links_g): @@ -161,14 +164,13 @@ def link_endpoints_shapes(endpoints_g, shapes_g, links_g): endpoints_g.remove((s2, TEMP.relevantShapes, o2)) -if __name__ == "__main__": - file_path = Path(__file__).parent.parent.parent / "test_data" / "custom_endpoints.json" - data = read_json(file_path) +def create_endpoint_rdf(endpoint_json: dict): + data = add_inverse_for_top_level(endpoint_json) g = Graph() create_endpoint_metadata(data, g) levels = process_relations(data) g2 = Graph() - results = process_levels(levels, g2) + process_levels(levels, g2) g3 = Graph() link_endpoints_shapes(g, g2, g3) complete = g + g2 + g3 @@ -176,3 +178,4 @@ def link_endpoints_shapes(endpoints_g, shapes_g, links_g): complete.bind("ex", EX) file_path = Path(__file__).parent.parent / "reference_data" / "endpoints" / "data_endpoints_custom" / "custom_endpoints.ttl" complete.serialize(destination=file_path, format="turtle") + diff --git a/prez/static/endpoint_config.html b/prez/static/endpoint_config.html new file mode 100644 index 00000000..7ea7b311 --- /dev/null +++ b/prez/static/endpoint_config.html @@ -0,0 +1,198 @@ + + + + Configure Endpoints + + + + +
+
+

Configure Endpoints

+ +
+

Instructions:

+
    +
  1. Configure endpoints here and submit. This will create an endpoints configuration file in prez/reference_data/endpoints/data_endpoints_custom/custom_endpoints.ttl
    2. +
  2. Set the following environment variables: +
      +
    1. CUSTOM_ENDPOINTS=true
      2. +
    2. ENDPOINT_STRUCTURE based on your custom endpoints e.g. ENDPOINT_STRUCTURE=["catalogs", "collections", "items"]
      3. +
    +
    3. +
  3. Restart Prez, confirm it is working as expected
    4. +
  4. Unset the CONFIGURATION_MODE environment variable (or set it to other than "true")
    5. +
+
+ +
+
+ +
+ +
+
+ +

Routes:

+ + +
+
+ +
+
+ +
+
+
+
+
+ + + + \ No newline at end of file From 4dfa3626887615fa8889d0f2a1ee3d49e9d777c9 Mon Sep 17 00:00:00 2001 From: david Date: Wed, 2 Oct 2024 16:52:37 +1000 Subject: [PATCH 3/7] Dynamic endpoints added using lifespan event as can require retrieval from remote repo. Move tests back to single dir. --- prez/app.py | 11 +- .../example_custom_endpoints.ttl | 43 ++++ .../default_endpoints.ttl | 88 ++++++++ .../profiles/ogc_records_profile.ttl | 2 +- prez/routers/custom_endpoints.py | 56 +++-- prez/services/app_service.py | 43 ++-- prez/services/generate_endpoint_rdf.py | 86 +++++--- prez/static/endpoint_config.html | 204 +++++++++++++----- .../TO_FIX_test_dd_profiles.py | 0 .../TO_FIX_test_endpoints_vocprez.py | 0 .../TO_FIX_test_search.py | 0 tests/__init__.py | 0 tests/{default_endpoints => }/_test_count.py | 0 tests/{default_endpoints => }/_test_cql.py | 0 .../_test_cql_fuseki.py | 0 .../_test_curie_generation.py | 0 tests/{default_endpoints => }/conftest.py | 10 +- .../cql-fuseki-config.ttl | 0 tests/custom_endpoints/conftest.py | 118 ---------- tests/default_endpoints/__init__.py | 0 .../test_alt_profiles.py | 0 tests/{default_endpoints => }/test_bnode.py | 2 +- tests/{default_endpoints => }/test_connegp.py | 0 .../{default_endpoints => }/test_cql_time.py | 6 +- .../test_curie_endpoint.py | 0 .../test_endpoints_cache.py | 0 .../test_endpoints_catprez.py | 0 .../test_endpoints_concept_hierarchy.py | 0 .../test_endpoints_management.py | 0 .../test_endpoints_object.py | 0 .../test_endpoints_ok.py | 0 .../test_endpoints_profiles.py | 0 .../test_endpoints_spaceprez.py | 0 .../test_geojson_to_wkt.py | 0 .../test_node_selection_shacl.py | 6 +- tests/{default_endpoints => }/test_ogc.py | 2 +- .../test_ogc_features_manual.py | 0 .../test_parse_datetimes.py | 0 .../test_property_selection_shacl.py | 0 .../test_query_construction.py | 0 .../test_redirect_endpoint.py | 0 .../test_remote_prefixes.py | 0 tests/{default_endpoints => }/test_search.py | 0 tests/{default_endpoints => }/test_sparql.py | 0 44 files changed, 398 insertions(+), 279 deletions(-) create mode 100644 prez/reference_data/endpoints/data_endpoints_custom/example_custom_endpoints.ttl create mode 100644 prez/reference_data/endpoints/data_endpoints_default/default_endpoints.ttl rename tests/{default_endpoints => }/TO_FIX_test_dd_profiles.py (100%) rename tests/{default_endpoints => }/TO_FIX_test_endpoints_vocprez.py (100%) rename tests/{default_endpoints => }/TO_FIX_test_search.py (100%) mode change 100755 => 100644 tests/__init__.py rename tests/{default_endpoints => }/_test_count.py (100%) rename tests/{default_endpoints => }/_test_cql.py (100%) rename tests/{default_endpoints => }/_test_cql_fuseki.py (100%) rename tests/{default_endpoints => }/_test_curie_generation.py (100%) rename tests/{default_endpoints => }/conftest.py (93%) rename tests/{default_endpoints => }/cql-fuseki-config.ttl (100%) delete mode 100755 tests/custom_endpoints/conftest.py delete mode 100644 tests/default_endpoints/__init__.py rename tests/{default_endpoints => }/test_alt_profiles.py (100%) rename tests/{default_endpoints => }/test_bnode.py (90%) rename tests/{default_endpoints => }/test_connegp.py (100%) rename tests/{default_endpoints => }/test_cql_time.py (91%) rename tests/{default_endpoints => }/test_curie_endpoint.py (100%) rename tests/{default_endpoints => }/test_endpoints_cache.py (100%) rename tests/{default_endpoints => }/test_endpoints_catprez.py (100%) rename tests/{default_endpoints => }/test_endpoints_concept_hierarchy.py (100%) rename tests/{default_endpoints => }/test_endpoints_management.py (100%) rename tests/{default_endpoints => }/test_endpoints_object.py (100%) rename tests/{default_endpoints => }/test_endpoints_ok.py (100%) rename tests/{default_endpoints => }/test_endpoints_profiles.py (100%) rename tests/{default_endpoints => }/test_endpoints_spaceprez.py (100%) rename tests/{default_endpoints => }/test_geojson_to_wkt.py (100%) rename tests/{default_endpoints => }/test_node_selection_shacl.py (92%) rename tests/{default_endpoints => }/test_ogc.py (96%) rename tests/{default_endpoints => }/test_ogc_features_manual.py (100%) rename tests/{default_endpoints => }/test_parse_datetimes.py (100%) rename tests/{default_endpoints => }/test_property_selection_shacl.py (100%) rename tests/{default_endpoints => }/test_query_construction.py (100%) rename tests/{default_endpoints => }/test_redirect_endpoint.py (100%) rename tests/{default_endpoints => }/test_remote_prefixes.py (100%) rename tests/{default_endpoints => }/test_search.py (100%) rename tests/{default_endpoints => }/test_sparql.py (100%) diff --git a/prez/app.py b/prez/app.py index feab4815..a37b84e2 100755 --- a/prez/app.py +++ b/prez/app.py @@ -126,6 +126,9 @@ async def lifespan(app: FastAPI): await load_system_data_to_oxigraph(app.state.pyoxi_system_store) await load_annotations_data_to_oxigraph(app.state.annotations_store) + # dynamic routes are either: custom routes if enabled, else default prez "data" routes are added dynamically + app.include_router(create_dynamic_router()) + yield # Shutdown @@ -181,18 +184,14 @@ def assemble_app( app.include_router(sparql_router) if _settings.configuration_mode: app.include_router(config_router) + app.mount("/static", StaticFiles(directory="static"), name="static") if _settings.enable_ogc_features: app.mount( - "/catalogs/{catalogId}/datasets/{datasetId}/features", + "/catalogs/{catalogId}/collections/{recordsCollectionId}/features", features_subapi, ) - if _settings.custom_endpoints: - app.include_router( - create_dynamic_router() - ) app.include_router(base_prez_router) app.include_router(identifier_router) - app.mount("/static", StaticFiles(directory="static"), name="static") app.openapi = partial( prez_open_api_metadata, title=title, diff --git a/prez/reference_data/endpoints/data_endpoints_custom/example_custom_endpoints.ttl b/prez/reference_data/endpoints/data_endpoints_custom/example_custom_endpoints.ttl new file mode 100644 index 00000000..337de634 --- /dev/null +++ b/prez/reference_data/endpoints/data_endpoints_custom/example_custom_endpoints.ttl @@ -0,0 +1,43 @@ +@prefix ex: . +@prefix ont: . +@prefix rdf: . +@prefix rdfs: . +@prefix schema: . +@prefix sh: . +@prefix skos: . +@prefix xsd: . + +ex:catalogs-listing a ont:ListingEndpoint , ont:DynamicEndpoint ; + rdfs:label "Catalogs Listing" ; + ont:apiPath "/catalogs" ; + ont:relevantShapes ex:shape-R0-HL1 . + +ex:catalogs-object a ont:ObjectEndpoint , ont:DynamicEndpoint ; + rdfs:label "Catalogs Object" ; + ont:apiPath "/catalogs/{catalogId}" ; + ont:relevantShapes ex:shape-R0-HL1 . + +ex:items-listing a ont:ListingEndpoint , ont:DynamicEndpoint ; + rdfs:label "Items Listing" ; + ont:apiPath "/catalogs/{catalogId}/items" ; + ont:relevantShapes ex:shape-R0-HL2 . + +ex:items-object a ont:ObjectEndpoint , ont:DynamicEndpoint ; + rdfs:label "Items Object" ; + ont:apiPath "/catalogs/{catalogId}/items/{itemId}" ; + ont:relevantShapes ex:shape-R0-HL2 . + +ex:shape-R0-HL1 a sh:NodeShape ; + sh:property [ sh:or ( [ sh:class ] [ sh:class schema:CreativeWork ] [ sh:class ] [ sh:class ] ) ; + sh:path skos:member ] ; + sh:targetClass skos:Collection ; + ont:hierarchyLevel 1 . + +ex:shape-R0-HL2 a sh:NodeShape ; + sh:property [ sh:class skos:Collection ; + sh:path [ sh:inversePath skos:member ] ] ; + sh:targetClass , + , + , + schema:CreativeWork ; + ont:hierarchyLevel 2 . diff --git a/prez/reference_data/endpoints/data_endpoints_default/default_endpoints.ttl b/prez/reference_data/endpoints/data_endpoints_default/default_endpoints.ttl new file mode 100644 index 00000000..907ea1bf --- /dev/null +++ b/prez/reference_data/endpoints/data_endpoints_default/default_endpoints.ttl @@ -0,0 +1,88 @@ +@prefix dcat: . +@prefix dcterms: . +@prefix ex: . +@prefix ont: . +@prefix rdf: . +@prefix rdfs: . +@prefix sh: . +@prefix skos: . +@prefix xsd: . + +ex:catalogs-listing a ont:DynamicEndpoint, + ont:ListingEndpoint ; + rdfs:label "Catalogs Listing" ; + ont:apiPath "/catalogs" ; + ont:relevantShapes ex:shape-R0-HL1 . + +ex:catalogs-object a ont:DynamicEndpoint, + ont:ObjectEndpoint ; + rdfs:label "Catalogs Object" ; + ont:apiPath "/catalogs/{catalogId}" ; + ont:relevantShapes ex:shape-R0-HL1 . + +ex:collections-listing a ont:DynamicEndpoint, + ont:ListingEndpoint ; + rdfs:label "Collections Listing" ; + ont:apiPath "/catalogs/{catalogId}/collections" ; + ont:relevantShapes ex:shape-R0-HL2 . + +ex:collections-object a ont:DynamicEndpoint, + ont:ObjectEndpoint ; + rdfs:label "Collections Object" ; + ont:apiPath "/catalogs/{catalogId}/collections/{recordsCollectionId}" ; + ont:relevantShapes ex:shape-R0-HL2 . + +ex:items-listing a ont:DynamicEndpoint, + ont:ListingEndpoint ; + rdfs:label "Items Listing" ; + ont:apiPath "/catalogs/{catalogId}/collections/{recordsCollectionId}/items" ; + ont:relevantShapes ex:shape-R0-HL3, + ex:shape-R0-HL3-1, + ex:shape-R0-HL3-2 . + +ex:items-object a ont:DynamicEndpoint, + ont:ObjectEndpoint ; + rdfs:label "Items Object" ; + ont:apiPath "/catalogs/{catalogId}/collections/{recordsCollectionId}/items/{itemId}" ; + ont:relevantShapes ex:shape-R0-HL3, + ex:shape-R0-HL3-1, + ex:shape-R0-HL3-2 . + +ex:shape-R0-HL1 a sh:NodeShape ; + sh:property [ sh:or ( [ sh:class skos:Collection ] [ sh:class dcat:Dataset ] [ sh:class skos:ConceptScheme ] ) ; + sh:path dcterms:hasPart ] ; + sh:targetClass dcat:Catalog ; + ont:hierarchyLevel 1 . + +ex:shape-R0-HL2 a sh:NodeShape ; + sh:property [ sh:class dcat:Catalog ; + sh:path [ sh:inversePath dcterms:hasPart ] ] ; + sh:targetClass skos:Collection, + skos:ConceptScheme, + dcat:Dataset ; + ont:hierarchyLevel 2 . + +ex:shape-R0-HL3 a sh:NodeShape ; + sh:property [ sh:class dcat:Catalog ; + sh:path ( skos:inScheme [ sh:inversePath dcterms:hasPart ] ) ], + [ sh:class skos:ConceptScheme ; + sh:path skos:inScheme ] ; + sh:targetClass skos:Concept ; + ont:hierarchyLevel 3 . + +ex:shape-R0-HL3-1 a sh:NodeShape ; + sh:property [ sh:class skos:Collection ; + sh:path [ sh:inversePath skos:member ] ], + [ sh:class dcat:Catalog ; + sh:path ( [ sh:inversePath skos:member ] [ sh:inversePath dcterms:hasPart ] ) ] ; + sh:targetClass skos:Concept ; + ont:hierarchyLevel 3 . + +ex:shape-R0-HL3-2 a sh:NodeShape ; + sh:property [ sh:class dcat:Dataset ; + sh:path [ sh:inversePath dcterms:hasPart ] ], + [ sh:class dcat:Catalog ; + sh:path ( [ sh:inversePath dcterms:hasPart ] [ sh:inversePath dcterms:hasPart ] ) ] ; + sh:targetClass dcat:Resource ; + ont:hierarchyLevel 3 . + diff --git a/prez/reference_data/profiles/ogc_records_profile.ttl b/prez/reference_data/profiles/ogc_records_profile.ttl index 072c29af..61b1eee2 100644 --- a/prez/reference_data/profiles/ogc_records_profile.ttl +++ b/prez/reference_data/profiles/ogc_records_profile.ttl @@ -109,7 +109,7 @@ prez:OGCItemProfile ], [ sh:maxCount 0 ; - sh:path dcterms:hasPart , rdfs:member ; + sh:path dcterms:hasPart , rdfs:member , skos:member ; ] ; shext:bnode-depth 2 ; altr-ext:constrainsClass diff --git a/prez/routers/custom_endpoints.py b/prez/routers/custom_endpoints.py index 83497703..5217df21 100644 --- a/prez/routers/custom_endpoints.py +++ b/prez/routers/custom_endpoints.py @@ -1,12 +1,13 @@ import logging from pathlib import Path as PLPath from typing import List - +import os from fastapi import APIRouter, Depends from fastapi import Path from rdflib import Graph, RDF, RDFS from sparql_grammar_pydantic import ConstructQuery - +from httpx import Client +from prez.cache import endpoints_graph_cache from prez.dependencies import ( get_data_repo, get_system_repo, @@ -18,6 +19,8 @@ get_endpoint_structure, generate_concept_hierarchy_query, ) +from urllib.parse import urlencode + from prez.models.query_params import QueryParams from prez.reference_data.prez_ns import ONT from prez.repositories import Repo @@ -27,15 +30,9 @@ from prez.services.query_generation.concept_hierarchy import ConceptHierarchyQuery from prez.services.query_generation.cql import CQLParser from prez.services.query_generation.shacl import NodeShape +import sys -log = logging.getLogger(__name__) - - -def load_routes() -> Graph: - g = Graph() - for file in (PLPath(__file__).parent.parent / "reference_data" / "endpoints" / "data_endpoints_custom").glob("*ttl"): - g.parse(file, format="ttl") - return g +logger = logging.getLogger(__name__) def create_path_param(name: str, description: str, example: str): @@ -72,6 +69,7 @@ async def dynamic_list_handler( query_params=query_params, original_endpoint_type=ONT["ListingEndpoint"], ) + return dynamic_list_handler elif route_type == "ObjectEndpoint": async def dynamic_object_handler( @@ -88,6 +86,7 @@ async def dynamic_object_handler( pmts=pmts, profile_nodeshape=profile_nodeshape, ) + return dynamic_object_handler @@ -98,24 +97,24 @@ def extract_path_params(path: str) -> List[str]: # Add routes dynamically to the router def add_routes(router: APIRouter): - g = load_routes() routes = [] - for s in g.subjects(predicate=RDF.type, object=ONT.ListingEndpoint): - route = { - "path": str(g.value(s, ONT.apiPath)), - "name": str(s), - "description": str(g.value(s, RDFS.label)), - "type": "ListingEndpoint", - } - routes.append(route) - for s in g.subjects(predicate=RDF.type, object=ONT.ObjectEndpoint): - route = { - "path": str(g.value(s, ONT.apiPath)), - "name": str(s), - "description": str(g.value(s, RDFS.label)), - "type": "ObjectEndpoint", - } - routes.append(route) + for s in endpoints_graph_cache.subjects(predicate=RDF.type, object=ONT.DynamicEndpoint): + if ONT.ListingEndpoint in endpoints_graph_cache.objects(subject=s, predicate=RDF.type): + route = { + "path": str(endpoints_graph_cache.value(s, ONT.apiPath)), + "name": str(s), + "description": str(endpoints_graph_cache.value(s, RDFS.label)), + "type": "ListingEndpoint", + } + routes.append(route) + elif ONT.ObjectEndpoint in endpoints_graph_cache.objects(subject=s, predicate=RDF.type): + route = { + "path": str(endpoints_graph_cache.value(s, ONT.apiPath)), + "name": str(s), + "description": str(endpoints_graph_cache.value(s, RDFS.label)), + "type": "ObjectEndpoint", + } + routes.append(route) for route in routes: path_param_names = extract_path_params(route["path"]) @@ -153,11 +152,10 @@ def add_routes(router: APIRouter): openapi_extra=openapi_extras ) - log.info(f"Added route: {route['path']} with path parameters: {path_param_names}") + logger.info(f"Added dynamic route: {route['path']}") def create_dynamic_router() -> APIRouter: - log.info("Adding Custom Endpoints") router = APIRouter(tags=["Custom Endpoints"]) add_routes(router) return router diff --git a/prez/services/app_service.py b/prez/services/app_service.py index 9a51790e..e38d7fa6 100755 --- a/prez/services/app_service.py +++ b/prez/services/app_service.py @@ -18,7 +18,6 @@ from prez.services.query_generation.count import startup_count_objects from prez.services.query_generation.prefixes import PrefixQuery - log = logging.getLogger(__name__) @@ -90,7 +89,7 @@ async def retrieve_remote_template_queries(repo: Repo): } """ _, results = await repo.send_queries([], [(None, query)]) - if results: + if results[0][1]: for result in results[0][1]: bn = BNode() query = result["query"]["value"] @@ -122,7 +121,7 @@ async def add_local_prefixes(repo): for f in (Path(__file__).parent.parent / "reference_data/prefixes").glob("*.ttl"): g = Graph().parse(f, format="turtle") local_i = await _add_prefixes_from_graph(g) - log.info(f"{local_i+1:,} prefixes bound from file {f.name}") + log.info(f"{local_i + 1:,} prefixes bound from file {f.name}") async def generate_prefixes(repo: Repo): @@ -160,9 +159,9 @@ async def generate_prefixes(repo: Repo): async def _add_prefixes_from_graph(g): i = 0 for i, (s, prefix) in enumerate( - g.subject_objects( - predicate=URIRef("http://purl.org/vocab/vann/preferredNamespacePrefix") - ) + g.subject_objects( + predicate=URIRef("http://purl.org/vocab/vann/preferredNamespacePrefix") + ) ): namespace = g.value( s, URIRef("http://purl.org/vocab/vann/preferredNamespaceUri") @@ -171,7 +170,7 @@ async def _add_prefixes_from_graph(g): return i -async def create_endpoints_graph(app_state) -> Graph: +async def create_endpoints_graph(app_state): endpoints_root = Path(__file__).parent.parent / "reference_data/endpoints" # OGC Features endpoints if app_state.settings.enable_ogc_features: @@ -179,35 +178,31 @@ async def create_endpoints_graph(app_state) -> Graph: endpoints_graph_cache.parse(f) # Custom data endpoints if app_state.settings.custom_endpoints: - for f in (endpoints_root / "data_endpoints_custom").glob("*.ttl"): - endpoints_graph_cache.parse(f) - log.info("Custom endpoints loaded") + # first try remote, if endpoints are found, use these + g = await get_remote_endpoint_definitions(app_state.repo) + if g: + endpoints_graph_cache.__iadd__(g) + else: + for f in (endpoints_root / "data_endpoints_custom").glob("*.ttl"): + endpoints_graph_cache.parse(f) + log.info("Custom endpoints loaded from local file") # Default data endpoints else: for f in (endpoints_root / "data_endpoints_default").glob("*.ttl"): endpoints_graph_cache.parse(f) - await get_remote_endpoint_definitions(app_state.repo) # Base endpoints for f in (endpoints_root / "base").glob("*.ttl"): endpoints_graph_cache.parse(f) - async def get_remote_endpoint_definitions(repo): - remote_endpoints_query = f""" -PREFIX ont: -CONSTRUCT {{ - ?endpoint ?p ?o. -}} -WHERE {{ - ?endpoint a ont:Endpoint; - ?p ?o. -}} - """ - g, _ = await repo.send_queries([remote_endpoints_query], []) + listing_ep_query = f"DESCRIBE ?ep {{ ?ep a {ONT['ListingEndpoint'].n3()} }}" + object_ep_query = f"DESCRIBE ?ep {{ ?ep a {ONT['ObjectEndpoint'].n3()} }}" + ep_nodeshape_query = f"DESCRIBE ?shape {{ ?shape {ONT['hierarchyLevel'].n3()} ?obj }}" + g, _ = await repo.send_queries([listing_ep_query, object_ep_query, ep_nodeshape_query], []) if len(g) > 0: - endpoints_graph_cache.__iadd__(g) log.info(f"Remote endpoint definition(s) found and added") + return g else: log.info("No remote endpoint definitions found") diff --git a/prez/services/generate_endpoint_rdf.py b/prez/services/generate_endpoint_rdf.py index fe737a03..d38b74f0 100644 --- a/prez/services/generate_endpoint_rdf.py +++ b/prez/services/generate_endpoint_rdf.py @@ -6,25 +6,25 @@ from prez.reference_data.prez_ns import ONT - - EX = Namespace("http://example.org/") TEMP = Namespace("http://temporary/") -def add_endpoint(g, endpoint_type, name, api_path, shapes_name, i): +def add_endpoint(g, endpoint_type, name, api_path, i, route_num): hl = (i + 2) // 2 endpoint_uri = EX[f"{name}-{endpoint_type}"] title_cased = f"{name.title()} {endpoint_type.title()}" g.add((endpoint_uri, RDF.type, ONT[f"{endpoint_type.title()}Endpoint"])) + g.add((endpoint_uri, RDF.type, ONT.DynamicEndpoint)) g.add((endpoint_uri, RDFS.label, Literal(title_cased))) g.add((endpoint_uri, ONT.apiPath, Literal(api_path))) - g.add((endpoint_uri, TEMP.relevantShapes, Literal(hl))) + g.add((endpoint_uri, TEMP.route_num, Literal(route_num))) + g.add((endpoint_uri, TEMP.hierarchy_level, Literal(hl))) def create_endpoint_metadata(data, g): - for route in data['routes']: + for route_num, route in enumerate(data['routes']): fullApiPath = route["fullApiPath"] components = fullApiPath.split("/")[1:] @@ -34,11 +34,13 @@ def create_endpoint_metadata(data, g): shapes_name = name.title() if i % 2 == 0: - add_endpoint(g, "listing", name, api_path, shapes_name, i) + add_endpoint(g, "listing", name, api_path, i, route_num) else: - add_endpoint(g, "object", name, api_path, shapes_name, i) + add_endpoint(g, "object", name, api_path, i, route_num) + def process_relations(data): + levels_list = [] for route in data['routes']: levels = {} for hier_rel in route["hierarchiesRelations"]: @@ -60,21 +62,24 @@ def process_relations(data): "klasses_from": {klass_from_key}, "klasses_to": {klass_to_key} } - return levels + levels_list.append(levels) + return levels_list -def process_levels(levels: dict, g: Graph): +def process_levels(levels: dict, g: Graph, route_num: int, shape_names: set): unique_suffix = 1 - shape_names = set() for i, (k, v) in enumerate(levels.items()): - proposed_shape_uri = EX[f"shape-{k[2][1]}"] + proposed_shape_uri = EX[f"shape-R{route_num}-HL{k[2][1]}"] if proposed_shape_uri not in shape_names: - shape_names.add(proposed_shape_uri) shape_uri = proposed_shape_uri + shape_names.add(shape_uri) else: - shape_uri = EX[f"shape-{k[2][1]}-{unique_suffix}"] + shape_uri = EX[f"shape-R{route_num}-HL{k[2][1]}-{unique_suffix}"] + shape_names.add(shape_uri) unique_suffix += 1 g.add((shape_uri, RDF.type, SH.NodeShape)) + g.add((shape_uri, TEMP.route_num, Literal(route_num))) + g.add((shape_uri, TEMP.hierarchy_level, Literal(k[2][1]))) g.add((shape_uri, ONT.hierarchyLevel, Literal(k[2][1]))) # hierarchyLevel = levelTo klasses_to = [] klasses_from = [] @@ -95,7 +100,12 @@ def process_levels(levels: dict, g: Graph): g.add((klass_bn, SH["class"], klass)) klass_bns.append(klass_bn) Collection(g, or_bn, klass_bns) - g.add((prop_bn, SH["path"], URIRef(k[3][1]))) + if k[0][1] == "outbound": + path_bn = BNode() + g.add((prop_bn, SH.path, path_bn)) + g.add((path_bn, SH.inversePath, URIRef(k[3][1]))) # relation + else: + g.add((prop_bn, SH.path, URIRef(k[3][1]))) elif k[1][1] < k[2][1]: # levelFrom < levelTo if k[0][1] == "outbound": path_bn = BNode() @@ -135,7 +145,6 @@ def process_levels(levels: dict, g: Graph): g.add((second_prop_bn, SH["class"], URIRef(tup[2][1]))) - def add_inverse_for_top_level(data): """ the RDF relation between the first and second endpoints is reused in reverse so endpoints can be defined based on @@ -157,25 +166,40 @@ def add_inverse_for_top_level(data): def link_endpoints_shapes(endpoints_g, shapes_g, links_g): - for s, p, o in shapes_g.triples((None, ONT.hierarchyLevel, None)): - for s2, p2, o2 in endpoints_g.triples((None, TEMP.relevantShapes, None)): - if o == o2: - links_g.add((s2, ONT.relevantShapes, s)) - endpoints_g.remove((s2, TEMP.relevantShapes, o2)) + for s_s in shapes_g.subjects(predicate=RDF.type, object=SH.NodeShape): + s_route_num = shapes_g.value(s_s, TEMP.route_num) + s_hl = shapes_g.value(s_s, TEMP.hierarchy_level) + for ep_s, _, _ in endpoints_g.triples_choices((None, RDF.type, [ONT.ListingEndpoint, ONT.ObjectEndpoint])): + ep_route_num = endpoints_g.value(ep_s, TEMP.route_num) + ep_hl = endpoints_g.value(ep_s, TEMP.hierarchy_level) + if (s_route_num == ep_route_num) and (s_hl == ep_hl): + links_g.add((ep_s, ONT.relevantShapes, s_s)) + # endpoints_g.remove((ep_s, TEMP.relevantShapes, o2)) + + +def cleanup_temp_preds(g): + for s, p, o in g.triples((None, TEMP.route_num, None)): + g.remove((s, p, o)) + for s, p, o in g.triples((None, TEMP.hierarchy_level, None)): + g.remove((s, p, o)) def create_endpoint_rdf(endpoint_json: dict): data = add_inverse_for_top_level(endpoint_json) g = Graph() create_endpoint_metadata(data, g) - levels = process_relations(data) - g2 = Graph() - process_levels(levels, g2) - g3 = Graph() - link_endpoints_shapes(g, g2, g3) - complete = g + g2 + g3 - complete.bind("ont", ONT) - complete.bind("ex", EX) - file_path = Path(__file__).parent.parent / "reference_data" / "endpoints" / "data_endpoints_custom" / "custom_endpoints.ttl" - complete.serialize(destination=file_path, format="turtle") - + levels_list = process_relations(data) + shape_names = set() + for route_num, levels in enumerate(levels_list): + g2 = Graph() + process_levels(levels, g2, route_num, shape_names) + g3 = Graph() + link_endpoints_shapes(g, g2, g3) + g += g2 + g += g3 + cleanup_temp_preds(g) + g.bind("ont", ONT) + g.bind("ex", EX) + file_path = Path( + __file__).parent.parent / "reference_data" / "endpoints" / "data_endpoints_custom" / "custom_endpoints.ttl" + g.serialize(destination=file_path, format="turtle") diff --git a/prez/static/endpoint_config.html b/prez/static/endpoint_config.html index 7ea7b311..0169ba03 100644 --- a/prez/static/endpoint_config.html +++ b/prez/static/endpoint_config.html @@ -2,45 +2,12 @@ Configure Endpoints - - +

Configure Endpoints

-

Instructions:

    @@ -56,7 +23,7 @@

    Instructions:

-
+
@@ -64,15 +31,19 @@

Instructions:

-

Routes:

- +
+ +
+ +
+
-
+
- +
@@ -83,12 +54,28 @@

Routes:

const routeContainer = document.querySelector('div#route-container'); let routeCount = 0; + function addRemoveButton(element, container, removeFunction) { + const removeBtn = document.createElement('button'); + removeBtn.className = 'button is-danger is-small'; + removeBtn.textContent = 'Remove'; + removeBtn.style.marginLeft = '10px'; + removeBtn.addEventListener('click', (e) => { + e.preventDefault(); + removeFunction(element, container); + }); + element.querySelector('h3, h4, h5').appendChild(removeBtn); + } + + function removeElement(element, container) { + container.removeChild(element); + } + document.querySelector('button.add-route').addEventListener('click', (e) => { e.preventDefault(); const newRoute = document.createElement('div'); - newRoute.classList.add('box', 'mt-4'); + newRoute.className = 'box'; newRoute.innerHTML = ` -

Route ${routeCount + 1}

+

Route ${routeCount + 1}

@@ -101,49 +88,57 @@

Route ${routeCount + 1}

- -
+
+
+ +
+
+
`; routeContainer.appendChild(newRoute); + addRemoveButton(newRoute, routeContainer, removeElement); routeCount++; const hierarchyRelationBtn = newRoute.querySelector('button.add-hierarchy-relation'); hierarchyRelationBtn.addEventListener('click', (e) => { e.preventDefault(); const newHierarchyRelation = document.createElement('div'); - newHierarchyRelation.classList.add('box'); + newHierarchyRelation.className = 'box'; newHierarchyRelation.innerHTML = ` -

Add Class Hierarchy

+

Add Class Hierarchy

- - Prez currently supports class hierarchies of up to three levels -
-
-
+
+
+ +
+
+
+
+
`; const hierarchyRelationsContainer = newRoute.querySelector(`div#hierarchy-relations-container-${routeCount - 1}`); hierarchyRelationsContainer.appendChild(newHierarchyRelation); + addRemoveButton(newHierarchyRelation, hierarchyRelationsContainer, removeElement); const classContainer = newHierarchyRelation.querySelector(`div#class-container-${routeCount}`); const relationContainer = newHierarchyRelation.querySelector(`div#relation-container-${routeCount}`); - const classBtn = newHierarchyRelation.querySelector('button.add-class'); - const limitMessage = newHierarchyRelation.querySelector('.limit-message'); let classCount = 0; + const classBtn = newHierarchyRelation.querySelector('button.add-class'); classBtn.addEventListener('click', (e) => { e.preventDefault(); if (classCount < 3) { classCount++; const newClass = document.createElement('div'); - newClass.className = 'class-item box'; + newClass.className = 'box class-item'; newClass.innerHTML = ` -

Class at Hierarchy Level ${classCount}

+
Class at Hierarchy Level ${classCount}
@@ -158,12 +153,19 @@

Class at Hierarchy Level ${cl

`; classContainer.appendChild(newClass); + addRemoveButton(newClass, classContainer, (element, container) => { + removeElement(element, container); + classCount--; + classBtn.disabled = false; + updateClassLevels(classContainer); + updateRelations(relationContainer, classCount); + }); if (classCount > 1) { const newRelation = document.createElement('div'); - newRelation.className = 'relation-item box'; + newRelation.className = 'box relation-item'; newRelation.innerHTML = ` -

Relation between Classes at Hierarchy Levels ${classCount - 1} and ${classCount}

+
Relation between Classes at Hierarchy Levels ${classCount - 1} and ${classCount}
@@ -183,14 +185,102 @@

Relation between Classes at Hierarchy Levels ${classCount -

`; relationContainer.appendChild(newRelation); + addRemoveButton(newRelation, relationContainer, removeElement); } if (classCount === 3) { classBtn.disabled = true; - limitMessage.style.display = 'inline'; } } }); + + function updateClassLevels(container) { + const classes = container.querySelectorAll('.class-item'); + classes.forEach((classItem, index) => { + const levelTag = classItem.querySelector('.tag'); + levelTag.textContent = index + 1; + }); + } + + function updateRelations(container, classCount) { + const relations = container.querySelectorAll('.relation-item'); + relations.forEach((relationItem, index) => { + const title = relationItem.querySelector('h5'); + title.textContent = `Relation between Classes at Hierarchy Levels ${index + 1} and ${index + 2}`; + }); + while (relations.length > classCount - 1) { + container.removeChild(relations[relations.length - 1]); + } + } + }); + }); + + document.getElementById('configForm').addEventListener('submit', function(e) { + e.preventDefault(); + const formData = new FormData(this); + + const config = { + configName: formData.get('configName'), + routes: [] + }; + + const routes = document.querySelectorAll('.route-container > div'); + routes.forEach(route => { + const routeData = { + name: route.querySelector('[name="name"]').value, + fullApiPath: route.querySelector('[name="fullApiPath"]').value, + hierarchiesRelations: [] + }; + + const hierarchyRelations = route.querySelectorAll('.hierarchy-relations-container > div'); + hierarchyRelations.forEach(hierarchyRelation => { + const hierarchyRelationData = { + name: hierarchyRelation.querySelector('[name="name"]').value, + hierarchy: [], + relations: [] + }; + + const classes = hierarchyRelation.querySelectorAll('.class-item'); + classes.forEach((classItem, index) => { + hierarchyRelationData.hierarchy.push({ + rdfClass: classItem.querySelector('[name="rdfClass"]').value, + className: classItem.querySelector('[name="className"]').value, + hierarchyLevel: index + 1 + }); + }); + + const relations = hierarchyRelation.querySelectorAll('.relation-item'); + relations.forEach((relationItem, index) => { + hierarchyRelationData.relations.push({ + levelFrom: index + 1, + levelTo: index + 2, + direction: relationItem.querySelector('[name="direction"]').value, + rdfPredicate: relationItem.querySelector('[name="rdfPredicate"]').value + }); + }); + + routeData.hierarchiesRelations.push(hierarchyRelationData); + }); + + config.routes.push(routeData); + }); + + // Post the data to /configure-endpoints + fetch('/configure-endpoints', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(config), + }) + .then(response => response.json()) + .then(data => { + console.log('Success:', data); + alert('Configuration submitted successfully!'); + }) + .catch((error) => { + console.error('Error:', error); + alert('An error occurred while submitting the configuration. Please try again.'); }); }); diff --git a/tests/default_endpoints/TO_FIX_test_dd_profiles.py b/tests/TO_FIX_test_dd_profiles.py similarity index 100% rename from tests/default_endpoints/TO_FIX_test_dd_profiles.py rename to tests/TO_FIX_test_dd_profiles.py diff --git a/tests/default_endpoints/TO_FIX_test_endpoints_vocprez.py b/tests/TO_FIX_test_endpoints_vocprez.py similarity index 100% rename from tests/default_endpoints/TO_FIX_test_endpoints_vocprez.py rename to tests/TO_FIX_test_endpoints_vocprez.py diff --git a/tests/default_endpoints/TO_FIX_test_search.py b/tests/TO_FIX_test_search.py similarity index 100% rename from tests/default_endpoints/TO_FIX_test_search.py rename to tests/TO_FIX_test_search.py diff --git a/tests/__init__.py b/tests/__init__.py old mode 100755 new mode 100644 diff --git a/tests/default_endpoints/_test_count.py b/tests/_test_count.py similarity index 100% rename from tests/default_endpoints/_test_count.py rename to tests/_test_count.py diff --git a/tests/default_endpoints/_test_cql.py b/tests/_test_cql.py similarity index 100% rename from tests/default_endpoints/_test_cql.py rename to tests/_test_cql.py diff --git a/tests/default_endpoints/_test_cql_fuseki.py b/tests/_test_cql_fuseki.py similarity index 100% rename from tests/default_endpoints/_test_cql_fuseki.py rename to tests/_test_cql_fuseki.py diff --git a/tests/default_endpoints/_test_curie_generation.py b/tests/_test_curie_generation.py similarity index 100% rename from tests/default_endpoints/_test_curie_generation.py rename to tests/_test_curie_generation.py diff --git a/tests/default_endpoints/conftest.py b/tests/conftest.py similarity index 93% rename from tests/default_endpoints/conftest.py rename to tests/conftest.py index f183164f..8ea99ef1 100755 --- a/tests/default_endpoints/conftest.py +++ b/tests/conftest.py @@ -22,24 +22,24 @@ from prez.repositories import Repo, PyoxigraphRepo -@pytest.fixture(scope="module") +@pytest.fixture(scope="session") def test_store() -> Store: # Create a new pyoxigraph Store store = Store() - for file in (Path(__file__).parent.parent.parent / "test_data").glob("*.ttl"): + for file in (Path(__file__).parent.parent / "test_data").glob("*.ttl"): store.load(file.read_bytes(), "text/turtle") return store -@pytest.fixture(scope="module") +@pytest.fixture(scope="session") def test_repo(test_store: Store) -> Repo: # Create a PyoxigraphQuerySender using the test_store return PyoxigraphRepo(test_store) -@pytest.fixture(scope="module") +@pytest.fixture(scope="session") def client(test_repo: Repo) -> TestClient: # Override the dependency to use the test_repo def override_get_repo(): @@ -60,7 +60,7 @@ def override_get_repo(): app.dependency_overrides.clear() -@pytest.fixture(scope="module") +@pytest.fixture(scope="session") def client_no_override() -> TestClient: app = assemble_app() diff --git a/tests/default_endpoints/cql-fuseki-config.ttl b/tests/cql-fuseki-config.ttl similarity index 100% rename from tests/default_endpoints/cql-fuseki-config.ttl rename to tests/cql-fuseki-config.ttl diff --git a/tests/custom_endpoints/conftest.py b/tests/custom_endpoints/conftest.py deleted file mode 100755 index 46a2576b..00000000 --- a/tests/custom_endpoints/conftest.py +++ /dev/null @@ -1,118 +0,0 @@ -import os - -from rdflib import Graph, URIRef, RDF -from rdflib.namespace import GEO -from starlette.routing import Mount - -# comment / uncomment for the CQL tests - cannot figure out how to get a different conftest picked up. -os.environ["SPARQL_REPO_TYPE"] = "pyoxigraph" - -# os.environ["SPARQL_ENDPOINT"] = "http://localhost:3030/dataset" -# os.environ["SPARQL_REPO_TYPE"] = "remote" -os.environ["ENABLE_SPARQL_ENDPOINT"] = "true" -os.environ["CUSTOM_ENDPOINTS"] = "true" - -from pathlib import Path - -import pytest -from fastapi.testclient import TestClient -from pyoxigraph.pyoxigraph import Store - -from prez.app import assemble_app -from prez.dependencies import get_data_repo -from prez.repositories import Repo, PyoxigraphRepo - - -@pytest.fixture(scope="module") -def test_store() -> Store: - # Create a new pyoxigraph Store - store = Store() - - for file in Path(__file__).parent.parent.glob("../test_data/*.ttl"): - store.load(file.read_bytes(), "text/turtle") - - return store - - -@pytest.fixture(scope="module") -def test_repo(test_store: Store) -> Repo: - # Create a PyoxigraphQuerySender using the test_store - return PyoxigraphRepo(test_store) - - -@pytest.fixture(scope="module") -def client(test_repo: Repo) -> TestClient: - # Override the dependency to use the test_repo - def override_get_repo(): - return test_repo - - app = assemble_app() - - app.dependency_overrides[get_data_repo] = override_get_repo - - for route in app.routes: - if isinstance(route, Mount): - route.app.dependency_overrides[get_data_repo] = override_get_repo - - with TestClient(app) as c: - yield c - - # Remove the override to ensure subsequent tests are unaffected - app.dependency_overrides.clear() - - -@pytest.fixture(scope="module") -def client_no_override() -> TestClient: - - app = assemble_app() - - with TestClient(app) as c: - yield c - - -@pytest.fixture() -def a_spaceprez_catalog_link(client): - r = client.get("/catalogs") - g = Graph().parse(data=r.text) - cat_uri = URIRef("https://example.com/spaceprez/SpacePrezCatalog") - link = g.value(cat_uri, URIRef(f"https://prez.dev/link", None)) - return link - - -@pytest.fixture() -def a_spaceprez_dataset_link(client, a_spaceprez_catalog_link): - r = client.get(f"{a_spaceprez_catalog_link}/collections") - g = Graph().parse(data=r.text) - ds_uri = URIRef("https://example.com/spaceprez/SpacePrezDataset") - link = g.value(ds_uri, URIRef(f"https://prez.dev/link", None)) - return link - - -@pytest.fixture() -def an_fc_link(client, a_spaceprez_dataset_link): - return f"{a_spaceprez_dataset_link}/features/collections/spcprz:FeatureCollection" - - -@pytest.fixture() -def a_feature_link(client, an_fc_link): - return f"{an_fc_link}/items/spcprz:Feature1" - - -@pytest.fixture() -def a_catprez_catalog_link(client): - # get link for first catalog - r = client.get("/catalogs") - g = Graph().parse(data=r.text) - member_uri = URIRef("https://example.com/CatalogOne") - link = g.value(member_uri, URIRef(f"https://prez.dev/link", None)) - return link - - -@pytest.fixture() -def a_resource_link(client, a_catprez_catalog_link): - r = client.get(a_catprez_catalog_link) - g = Graph().parse(data=r.text) - links = g.objects(subject=None, predicate=URIRef(f"https://prez.dev/link")) - for link in links: - if link != a_catprez_catalog_link: - return link diff --git a/tests/default_endpoints/__init__.py b/tests/default_endpoints/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/default_endpoints/test_alt_profiles.py b/tests/test_alt_profiles.py similarity index 100% rename from tests/default_endpoints/test_alt_profiles.py rename to tests/test_alt_profiles.py diff --git a/tests/default_endpoints/test_bnode.py b/tests/test_bnode.py similarity index 90% rename from tests/default_endpoints/test_bnode.py rename to tests/test_bnode.py index e5c2d807..b6064ebc 100755 --- a/tests/default_endpoints/test_bnode.py +++ b/tests/test_bnode.py @@ -18,7 +18,7 @@ ], ) def test_bnode_depth(input_file: str, iri: str, expected_depth: int) -> None: - file = Path(__file__).parent.parent.parent / "test_data" / input_file + file = Path(__file__).parent.parent / "test_data" / input_file graph = Graph() graph.parse(file) diff --git a/tests/default_endpoints/test_connegp.py b/tests/test_connegp.py similarity index 100% rename from tests/default_endpoints/test_connegp.py rename to tests/test_connegp.py diff --git a/tests/default_endpoints/test_cql_time.py b/tests/test_cql_time.py similarity index 91% rename from tests/default_endpoints/test_cql_time.py rename to tests/test_cql_time.py index 40f0c821..4d40790f 100755 --- a/tests/default_endpoints/test_cql_time.py +++ b/tests/test_cql_time.py @@ -42,16 +42,16 @@ ) def test_time_funcs(cql_json_filename, output_query_filename): cql_json_path = ( - Path(__file__).parent.parent.parent / f"test_data/cql/input/{cql_json_filename}" + Path(__file__).parent.parent / f"test_data/cql/input/{cql_json_filename}" ) cql_json = json.loads(cql_json_path.read_text()) reference_query = ( - Path(__file__).parent.parent.parent + Path(__file__).parent.parent / f"test_data/cql/expected_generated_queries/{output_query_filename}" ).read_text() context = json.load( ( - Path(__file__).parent.parent.parent + Path(__file__).parent.parent / "prez/reference_data/cql/default_context.json" ).open() ) diff --git a/tests/default_endpoints/test_curie_endpoint.py b/tests/test_curie_endpoint.py similarity index 100% rename from tests/default_endpoints/test_curie_endpoint.py rename to tests/test_curie_endpoint.py diff --git a/tests/default_endpoints/test_endpoints_cache.py b/tests/test_endpoints_cache.py similarity index 100% rename from tests/default_endpoints/test_endpoints_cache.py rename to tests/test_endpoints_cache.py diff --git a/tests/default_endpoints/test_endpoints_catprez.py b/tests/test_endpoints_catprez.py similarity index 100% rename from tests/default_endpoints/test_endpoints_catprez.py rename to tests/test_endpoints_catprez.py diff --git a/tests/default_endpoints/test_endpoints_concept_hierarchy.py b/tests/test_endpoints_concept_hierarchy.py similarity index 100% rename from tests/default_endpoints/test_endpoints_concept_hierarchy.py rename to tests/test_endpoints_concept_hierarchy.py diff --git a/tests/default_endpoints/test_endpoints_management.py b/tests/test_endpoints_management.py similarity index 100% rename from tests/default_endpoints/test_endpoints_management.py rename to tests/test_endpoints_management.py diff --git a/tests/default_endpoints/test_endpoints_object.py b/tests/test_endpoints_object.py similarity index 100% rename from tests/default_endpoints/test_endpoints_object.py rename to tests/test_endpoints_object.py diff --git a/tests/default_endpoints/test_endpoints_ok.py b/tests/test_endpoints_ok.py similarity index 100% rename from tests/default_endpoints/test_endpoints_ok.py rename to tests/test_endpoints_ok.py diff --git a/tests/default_endpoints/test_endpoints_profiles.py b/tests/test_endpoints_profiles.py similarity index 100% rename from tests/default_endpoints/test_endpoints_profiles.py rename to tests/test_endpoints_profiles.py diff --git a/tests/default_endpoints/test_endpoints_spaceprez.py b/tests/test_endpoints_spaceprez.py similarity index 100% rename from tests/default_endpoints/test_endpoints_spaceprez.py rename to tests/test_endpoints_spaceprez.py diff --git a/tests/default_endpoints/test_geojson_to_wkt.py b/tests/test_geojson_to_wkt.py similarity index 100% rename from tests/default_endpoints/test_geojson_to_wkt.py rename to tests/test_geojson_to_wkt.py diff --git a/tests/default_endpoints/test_node_selection_shacl.py b/tests/test_node_selection_shacl.py similarity index 92% rename from tests/default_endpoints/test_node_selection_shacl.py rename to tests/test_node_selection_shacl.py index 9b895e30..1ebe1e4b 100755 --- a/tests/default_endpoints/test_node_selection_shacl.py +++ b/tests/test_node_selection_shacl.py @@ -9,13 +9,13 @@ from sparql_grammar_pydantic import Var endpoints_graph = Graph().parse( - Path(__file__).parent.parent.parent + Path(__file__).parent.parent / "prez/reference_data/endpoints/data_endpoints_default/default_endpoints.ttl", format="turtle", ) -@pytest.mark.parametrize("nodeshape_uri", ["http://example.org/shape-2"]) +@pytest.mark.parametrize("nodeshape_uri", ["http://example.org/shape-R0-HL2"]) def test_nodeshape_parsing(nodeshape_uri): ns = NodeShape( uri=URIRef(nodeshape_uri), @@ -33,7 +33,7 @@ def test_nodeshape_parsing(nodeshape_uri): @pytest.mark.parametrize( "nodeshape_uri", - ["http://example.org/shape-3"], + ["http://example.org/shape-R0-HL3-2"], ) def test_nodeshape_to_grammar(nodeshape_uri): ns = NodeShape( diff --git a/tests/default_endpoints/test_ogc.py b/tests/test_ogc.py similarity index 96% rename from tests/default_endpoints/test_ogc.py rename to tests/test_ogc.py index 043c98ff..280ee4d9 100644 --- a/tests/default_endpoints/test_ogc.py +++ b/tests/test_ogc.py @@ -16,7 +16,7 @@ def test_store() -> Store: # Create a new pyoxigraph Store store = Store() - file = Path(__file__).parent.parent.parent / "test_data/ogc_features.ttl" + file = Path(__file__).parent.parent / "test_data/ogc_features.ttl" store.load(file.read_bytes(), "text/turtle") return store diff --git a/tests/default_endpoints/test_ogc_features_manual.py b/tests/test_ogc_features_manual.py similarity index 100% rename from tests/default_endpoints/test_ogc_features_manual.py rename to tests/test_ogc_features_manual.py diff --git a/tests/default_endpoints/test_parse_datetimes.py b/tests/test_parse_datetimes.py similarity index 100% rename from tests/default_endpoints/test_parse_datetimes.py rename to tests/test_parse_datetimes.py diff --git a/tests/default_endpoints/test_property_selection_shacl.py b/tests/test_property_selection_shacl.py similarity index 100% rename from tests/default_endpoints/test_property_selection_shacl.py rename to tests/test_property_selection_shacl.py diff --git a/tests/default_endpoints/test_query_construction.py b/tests/test_query_construction.py similarity index 100% rename from tests/default_endpoints/test_query_construction.py rename to tests/test_query_construction.py diff --git a/tests/default_endpoints/test_redirect_endpoint.py b/tests/test_redirect_endpoint.py similarity index 100% rename from tests/default_endpoints/test_redirect_endpoint.py rename to tests/test_redirect_endpoint.py diff --git a/tests/default_endpoints/test_remote_prefixes.py b/tests/test_remote_prefixes.py similarity index 100% rename from tests/default_endpoints/test_remote_prefixes.py rename to tests/test_remote_prefixes.py diff --git a/tests/default_endpoints/test_search.py b/tests/test_search.py similarity index 100% rename from tests/default_endpoints/test_search.py rename to tests/test_search.py diff --git a/tests/default_endpoints/test_sparql.py b/tests/test_sparql.py similarity index 100% rename from tests/default_endpoints/test_sparql.py rename to tests/test_sparql.py From b0a5010e1ff99d39d63c1bcd14728a2348bfa917 Mon Sep 17 00:00:00 2001 From: david Date: Fri, 4 Oct 2024 11:23:59 +1000 Subject: [PATCH 4/7] Black + Add Endpoint Configuration Readme --- prez/models/endpoint_config.py | 23 +++++----- prez/routers/custom_endpoints.py | 60 ++++++++++++++++---------- prez/routers/management.py | 23 +++++----- prez/services/app_service.py | 14 +++--- prez/services/connegp_service.py | 22 ++++++++-- prez/services/generate_endpoint_rdf.py | 47 ++++++++++++++------ tests/test_bnode.py | 2 - 7 files changed, 124 insertions(+), 67 deletions(-) diff --git a/prez/models/endpoint_config.py b/prez/models/endpoint_config.py index c959aa32..b283cc37 100644 --- a/prez/models/endpoint_config.py +++ b/prez/models/endpoint_config.py @@ -42,11 +42,14 @@ class HierarchyRelation(BaseModel): @classmethod @field_validator("relations") - def validate_relations_count(cls, v: List[Relation], values: dict) -> List[Relation]: - hierarchy = values.get('hierarchy') + def validate_relations_count( + cls, v: List[Relation], values: dict + ) -> List[Relation]: + hierarchy = values.get("hierarchy") if hierarchy and len(v) != len(hierarchy) - 1: raise ValueError( - f"Number of relations ({len(v)}) should be one less than the number of hierarchy items ({len(hierarchy)})") + f"Number of relations ({len(v)}) should be one less than the number of hierarchy items ({len(hierarchy)})" + ) return v @@ -73,24 +76,24 @@ class RootModel(BaseModel): { "rdfClass": "http://www.w3.org/ns/dcat#Catalog", "className": "Catalog", - "hierarchyLevel": 1 + "hierarchyLevel": 1, }, { "rdfClass": "http://www.w3.org/ns/dcat#Dataset", "className": "DCAT Dataset", - "hierarchyLevel": 2 - } + "hierarchyLevel": 2, + }, ], "relations": [ { "levelFrom": 1, "levelTo": 2, "direction": "outbound", - "rdfPredicate": "http://purl.org/dc/terms/hasPart" + "rdfPredicate": "http://purl.org/dc/terms/hasPart", } - ] + ], } - ] + ], } - ] + ], } diff --git a/prez/routers/custom_endpoints.py b/prez/routers/custom_endpoints.py index 5217df21..bda0c222 100644 --- a/prez/routers/custom_endpoints.py +++ b/prez/routers/custom_endpoints.py @@ -42,19 +42,20 @@ def create_path_param(name: str, description: str, example: str): # Dynamic route handler def create_dynamic_route_handler(route_type: str): if route_type == "ListingEndpoint": + async def dynamic_list_handler( - query_params: QueryParams = Depends(), - endpoint_nodeshape: NodeShape = Depends(get_endpoint_nodeshapes), - pmts: NegotiatedPMTs = Depends(get_negotiated_pmts), - endpoint_structure: tuple[str, ...] = Depends(get_endpoint_structure), - profile_nodeshape: NodeShape = Depends(get_profile_nodeshape), - cql_parser: CQLParser = Depends(cql_get_parser_dependency), - search_query: ConstructQuery = Depends(generate_search_query), - concept_hierarchy_query: ConceptHierarchyQuery = Depends( - generate_concept_hierarchy_query - ), - data_repo: Repo = Depends(get_data_repo), - system_repo: Repo = Depends(get_system_repo), + query_params: QueryParams = Depends(), + endpoint_nodeshape: NodeShape = Depends(get_endpoint_nodeshapes), + pmts: NegotiatedPMTs = Depends(get_negotiated_pmts), + endpoint_structure: tuple[str, ...] = Depends(get_endpoint_structure), + profile_nodeshape: NodeShape = Depends(get_profile_nodeshape), + cql_parser: CQLParser = Depends(cql_get_parser_dependency), + search_query: ConstructQuery = Depends(generate_search_query), + concept_hierarchy_query: ConceptHierarchyQuery = Depends( + generate_concept_hierarchy_query + ), + data_repo: Repo = Depends(get_data_repo), + system_repo: Repo = Depends(get_system_repo), ): return await listing_function( data_repo=data_repo, @@ -72,12 +73,13 @@ async def dynamic_list_handler( return dynamic_list_handler elif route_type == "ObjectEndpoint": + async def dynamic_object_handler( - pmts: NegotiatedPMTs = Depends(get_negotiated_pmts), - endpoint_structure: tuple[str, ...] = Depends(get_endpoint_structure), - profile_nodeshape: NodeShape = Depends(get_profile_nodeshape), - data_repo: Repo = Depends(get_data_repo), - system_repo: Repo = Depends(get_system_repo), + pmts: NegotiatedPMTs = Depends(get_negotiated_pmts), + endpoint_structure: tuple[str, ...] = Depends(get_endpoint_structure), + profile_nodeshape: NodeShape = Depends(get_profile_nodeshape), + data_repo: Repo = Depends(get_data_repo), + system_repo: Repo = Depends(get_system_repo), ): return await object_function( data_repo=data_repo, @@ -92,14 +94,22 @@ async def dynamic_object_handler( # Extract path parameters from the path def extract_path_params(path: str) -> List[str]: - return [part[1:-1] for part in path.split("/") if part.startswith("{") and part.endswith("}")] + return [ + part[1:-1] + for part in path.split("/") + if part.startswith("{") and part.endswith("}") + ] # Add routes dynamically to the router def add_routes(router: APIRouter): routes = [] - for s in endpoints_graph_cache.subjects(predicate=RDF.type, object=ONT.DynamicEndpoint): - if ONT.ListingEndpoint in endpoints_graph_cache.objects(subject=s, predicate=RDF.type): + for s in endpoints_graph_cache.subjects( + predicate=RDF.type, object=ONT.DynamicEndpoint + ): + if ONT.ListingEndpoint in endpoints_graph_cache.objects( + subject=s, predicate=RDF.type + ): route = { "path": str(endpoints_graph_cache.value(s, ONT.apiPath)), "name": str(s), @@ -107,7 +117,9 @@ def add_routes(router: APIRouter): "type": "ListingEndpoint", } routes.append(route) - elif ONT.ObjectEndpoint in endpoints_graph_cache.objects(subject=s, predicate=RDF.type): + elif ONT.ObjectEndpoint in endpoints_graph_cache.objects( + subject=s, predicate=RDF.type + ): route = { "path": str(endpoints_graph_cache.value(s, ONT.apiPath)), "name": str(s), @@ -121,7 +133,9 @@ def add_routes(router: APIRouter): # Create path parameters using FastAPI's Path path_params = { - param: create_path_param(param, f"Path parameter: {param}", f"example_{param}") + param: create_path_param( + param, f"Path parameter: {param}", f"example_{param}" + ) for param in path_param_names } @@ -149,7 +163,7 @@ def add_routes(router: APIRouter): endpoint=endpoint, methods=["GET"], description=route["description"], - openapi_extra=openapi_extras + openapi_extra=openapi_extras, ) logger.info(f"Added dynamic route: {route['path']}") diff --git a/prez/routers/management.py b/prez/routers/management.py index a5552581..86fa6e6b 100755 --- a/prez/routers/management.py +++ b/prez/routers/management.py @@ -86,7 +86,7 @@ async def return_tbox_cache(request: Request): pred_obj = pickle.loads(pred_obj_bytes) for pred, obj in pred_obj: if ( - pred_obj + pred_obj ): # cache entry for a URI can be empty - i.e. no annotations found for URI # Add the expanded triple (subject, predicate, object) to 'annotations_g' cache_g.add((subject, pred, obj)) @@ -123,9 +123,9 @@ async def return_annotation_predicates(): @router.get("/prefixes", summary="Show prefixes known to prez") async def show_prefixes( - mediatype: Optional[NonAnnotatedRDFMediaType | JSONMediaType] = Query( - default=NonAnnotatedRDFMediaType.TURTLE, alias="_mediatype" - ) + mediatype: Optional[NonAnnotatedRDFMediaType | JSONMediaType] = Query( + default=NonAnnotatedRDFMediaType.TURTLE, alias="_mediatype" + ) ): """Returns the prefixes known to prez""" mediatype_str = str(mediatype.value) @@ -142,16 +142,15 @@ async def show_prefixes( return StreamingResponse(content=content, media_type=mediatype_str) - @config_router.post("/configure-endpoints", summary="Configuration") -async def submit_config(config: RootModel = Body( - ..., - examples=[configure_endpoings_example] -) +async def submit_config( + config: RootModel = Body(..., examples=[configure_endpoings_example]) ): try: create_endpoint_rdf(config.model_dump()) - return {"message": f"Configuration received successfully. {len(config.routes)} routes processed."} + return { + "message": f"Configuration received successfully. {len(config.routes)} routes processed." + } except ValidationError as e: raise HTTPException(status_code=400, detail=str(e)) @@ -159,4 +158,6 @@ async def submit_config(config: RootModel = Body( @config_router.get("/configure-endpoints") async def open_config_page(): """Redirects to the endpoint configuration page""" - return Response(status_code=302, headers={"Location": "/static/endpoint_config.html"}) + return Response( + status_code=302, headers={"Location": "/static/endpoint_config.html"} + ) diff --git a/prez/services/app_service.py b/prez/services/app_service.py index e38d7fa6..6f657977 100755 --- a/prez/services/app_service.py +++ b/prez/services/app_service.py @@ -159,9 +159,9 @@ async def generate_prefixes(repo: Repo): async def _add_prefixes_from_graph(g): i = 0 for i, (s, prefix) in enumerate( - g.subject_objects( - predicate=URIRef("http://purl.org/vocab/vann/preferredNamespacePrefix") - ) + g.subject_objects( + predicate=URIRef("http://purl.org/vocab/vann/preferredNamespacePrefix") + ) ): namespace = g.value( s, URIRef("http://purl.org/vocab/vann/preferredNamespaceUri") @@ -198,8 +198,12 @@ async def create_endpoints_graph(app_state): async def get_remote_endpoint_definitions(repo): listing_ep_query = f"DESCRIBE ?ep {{ ?ep a {ONT['ListingEndpoint'].n3()} }}" object_ep_query = f"DESCRIBE ?ep {{ ?ep a {ONT['ObjectEndpoint'].n3()} }}" - ep_nodeshape_query = f"DESCRIBE ?shape {{ ?shape {ONT['hierarchyLevel'].n3()} ?obj }}" - g, _ = await repo.send_queries([listing_ep_query, object_ep_query, ep_nodeshape_query], []) + ep_nodeshape_query = ( + f"DESCRIBE ?shape {{ ?shape {ONT['hierarchyLevel'].n3()} ?obj }}" + ) + g, _ = await repo.send_queries( + [listing_ep_query, object_ep_query, ep_nodeshape_query], [] + ) if len(g) > 0: log.info(f"Remote endpoint definition(s) found and added") return g diff --git a/prez/services/connegp_service.py b/prez/services/connegp_service.py index e15b12de..5e35c91f 100755 --- a/prez/services/connegp_service.py +++ b/prez/services/connegp_service.py @@ -208,9 +208,23 @@ async def _get_available(self) -> list[dict]: ] if not available: if self.listing: - return [{"profile": URIRef("https://w3id.org/profile/mem"), "title": "Members", "mediatype": "text/anot+turtle", "class": "http://www.w3.org/2000/01/rdf-schema#Resource"}] + return [ + { + "profile": URIRef("https://w3id.org/profile/mem"), + "title": "Members", + "mediatype": "text/anot+turtle", + "class": "http://www.w3.org/2000/01/rdf-schema#Resource", + } + ] else: - return [{"profile": URIRef("https://prez.dev/profile/open-object"), "title": "Open profile", "mediatype": "text/anot+turtle", "class": "http://www.w3.org/2000/01/rdf-schema#Resource"}] + return [ + { + "profile": URIRef("https://prez.dev/profile/open-object"), + "title": "Open profile", + "mediatype": "text/anot+turtle", + "class": "http://www.w3.org/2000/01/rdf-schema#Resource", + } + ] return available def generate_response_headers(self) -> dict: @@ -238,7 +252,9 @@ def generate_response_headers(self) -> dict: def _compose_select_query(self) -> str: prez = Namespace("https://prez.dev/") profile_class = prez.ListingProfile if self.listing else prez.ObjectProfile - query_klasses = set(endpoints_graph_cache.objects(subject=None, predicate=SH.targetClass)) + query_klasses = set( + endpoints_graph_cache.objects(subject=None, predicate=SH.targetClass) + ) if self.requested_profiles: requested_profile = self.requested_profiles[0][0] else: diff --git a/prez/services/generate_endpoint_rdf.py b/prez/services/generate_endpoint_rdf.py index d38b74f0..36b8c871 100644 --- a/prez/services/generate_endpoint_rdf.py +++ b/prez/services/generate_endpoint_rdf.py @@ -24,7 +24,7 @@ def add_endpoint(g, endpoint_type, name, api_path, i, route_num): def create_endpoint_metadata(data, g): - for route_num, route in enumerate(data['routes']): + for route_num, route in enumerate(data["routes"]): fullApiPath = route["fullApiPath"] components = fullApiPath.split("/")[1:] @@ -41,18 +41,24 @@ def create_endpoint_metadata(data, g): def process_relations(data): levels_list = [] - for route in data['routes']: + for route in data["routes"]: levels = {} for hier_rel in route["hierarchiesRelations"]: hierarchy_dict = {h["hierarchyLevel"]: h for h in hier_rel["hierarchy"]} for relation in hier_rel["relations"]: - rel_key = tuple(sorted(relation.items())) # Sort items before creating tuple + rel_key = tuple( + sorted(relation.items()) + ) # Sort items before creating tuple level_from = relation["levelFrom"] level_to = relation["levelTo"] klass_from = hierarchy_dict[level_from] klass_to = hierarchy_dict[level_to] - klass_from_key = tuple(sorted(klass_from.items())) # Sort items before creating tuple - klass_to_key = tuple(sorted(klass_to.items())) # Sort items before creating tuple + klass_from_key = tuple( + sorted(klass_from.items()) + ) # Sort items before creating tuple + klass_to_key = tuple( + sorted(klass_to.items()) + ) # Sort items before creating tuple if rel_key in levels: levels[rel_key]["klasses_from"].add(klass_from_key) @@ -60,7 +66,7 @@ def process_relations(data): else: levels[rel_key] = { "klasses_from": {klass_from_key}, - "klasses_to": {klass_to_key} + "klasses_to": {klass_to_key}, } levels_list.append(levels) return levels_list @@ -80,7 +86,9 @@ def process_levels(levels: dict, g: Graph, route_num: int, shape_names: set): g.add((shape_uri, RDF.type, SH.NodeShape)) g.add((shape_uri, TEMP.route_num, Literal(route_num))) g.add((shape_uri, TEMP.hierarchy_level, Literal(k[2][1]))) - g.add((shape_uri, ONT.hierarchyLevel, Literal(k[2][1]))) # hierarchyLevel = levelTo + g.add( + (shape_uri, ONT.hierarchyLevel, Literal(k[2][1])) + ) # hierarchyLevel = levelTo klasses_to = [] klasses_from = [] for tup in v["klasses_to"]: @@ -136,7 +144,9 @@ def process_levels(levels: dict, g: Graph, route_num: int, shape_names: set): list_comps.append(URIRef(k[3][1])) if second_rel[0][1] == "outbound": inverse_bn = BNode() - g.add((inverse_bn, SH.inversePath, URIRef(second_rel[3][1]))) # relation + g.add( + (inverse_bn, SH.inversePath, URIRef(second_rel[3][1])) + ) # relation list_comps.append(inverse_bn) else: list_comps.append(URIRef(second_rel[3][1])) @@ -151,7 +161,7 @@ def add_inverse_for_top_level(data): their relation to each other rather than needing to say put the class of objects at the top level in some arbitrary collection """ - for route in data['routes']: + for route in data["routes"]: for hr in route["hierarchiesRelations"]: for relation in hr["relations"]: if relation["levelFrom"] == 1 and relation["levelTo"] == 2: @@ -159,7 +169,11 @@ def add_inverse_for_top_level(data): "levelFrom": 2, "levelTo": 1, "rdfPredicate": relation["rdfPredicate"], - "direction": "inbound" if relation["direction"] == "outbound" else "outbound" + "direction": ( + "inbound" + if relation["direction"] == "outbound" + else "outbound" + ), } hr["relations"].append(inverted_relation) return data @@ -169,7 +183,9 @@ def link_endpoints_shapes(endpoints_g, shapes_g, links_g): for s_s in shapes_g.subjects(predicate=RDF.type, object=SH.NodeShape): s_route_num = shapes_g.value(s_s, TEMP.route_num) s_hl = shapes_g.value(s_s, TEMP.hierarchy_level) - for ep_s, _, _ in endpoints_g.triples_choices((None, RDF.type, [ONT.ListingEndpoint, ONT.ObjectEndpoint])): + for ep_s, _, _ in endpoints_g.triples_choices( + (None, RDF.type, [ONT.ListingEndpoint, ONT.ObjectEndpoint]) + ): ep_route_num = endpoints_g.value(ep_s, TEMP.route_num) ep_hl = endpoints_g.value(ep_s, TEMP.hierarchy_level) if (s_route_num == ep_route_num) and (s_hl == ep_hl): @@ -200,6 +216,11 @@ def create_endpoint_rdf(endpoint_json: dict): cleanup_temp_preds(g) g.bind("ont", ONT) g.bind("ex", EX) - file_path = Path( - __file__).parent.parent / "reference_data" / "endpoints" / "data_endpoints_custom" / "custom_endpoints.ttl" + file_path = ( + Path(__file__).parent.parent + / "reference_data" + / "endpoints" + / "data_endpoints_custom" + / "custom_endpoints.ttl" + ) g.serialize(destination=file_path, format="turtle") diff --git a/tests/test_bnode.py b/tests/test_bnode.py index b6064ebc..f20bc118 100755 --- a/tests/test_bnode.py +++ b/tests/test_bnode.py @@ -6,8 +6,6 @@ from prez.bnode import get_bnode_depth - - @pytest.mark.parametrize( "input_file, iri, expected_depth", [ From c2848bf07aceba56864317fafbb9ac029468d17c Mon Sep 17 00:00:00 2001 From: david Date: Fri, 4 Oct 2024 14:53:29 +1000 Subject: [PATCH 5/7] Readme updates. Config page tweaks. --- README-Custom-Endpoints.md | 41 +++++ README-OGC-Features.md | 1 - prez/static/endpoint_config.html | 257 ++++++++++++++++--------------- 3 files changed, 175 insertions(+), 124 deletions(-) create mode 100644 README-Custom-Endpoints.md diff --git a/README-Custom-Endpoints.md b/README-Custom-Endpoints.md new file mode 100644 index 00000000..a8d07617 --- /dev/null +++ b/README-Custom-Endpoints.md @@ -0,0 +1,41 @@ +# Custom Data Endpoint Configuration + +Prez allows the configuration of custom endpoints. + +That is, you can specify the route structure e.g. + +`/catalogs/{catalogId}/products/{productId}` + +And then specify which classes these endpoints deliver, and what the RDF relations between those classes are. + +## Set up instructions +To set up custom endpoints: +1. Set the CONFIGURATION_MODE environment variable to "true" +2. Go to the `/configure-endpoints` page and complete the form; submit +3. Set the CUSTOM_ENDPOINTS environment variable to "true" +4. Restart Prez. You should see the dynamic endpoints being created in the logs. +5. Confirm your endpoints are working as expected. + +Once working as expected: +1. Copy the "custom_endpoints.ttl" file from `prez/reference_data/data_endpoints_custom` to the remote backend (e.g. triplestore) + > Prez will preferentially use the custom endpoints specified in the triplestore over the ones specified in the local file. +2. Set the CONFIGURATION_MODE environment variable to "false" + +## Limitations +The following limitations apply at present: +- The endpoint structure must be specified in the config to match what is input through the form. +related to this: +- Only one route can be specified (though multiple class hierarchies which use that one route can be specified) +i.e. you can specify +`/catalogs/{catalogId}/products/{productId}` +but not +`/catalogs/{catalogId}/products/{productId}` +and +`/datasets/{datasetId}/items/{itemsId} +on the one prez instance. +- This limitation is only due to link generation, which looks up the (currently) single endpoint structure variable in the config file. +- This should be resolvable with a small amount of work. At link generation time, an endpoint nodeshape is in context, and endpoint nodeshapes are mapped to a route structure. + +- The number of hierarchy levels within a route must be two or three (i.e. 2 or 3 levels of classes = 4-6 listing/object endpoints) + - The lower limit of two is because prez uses the relationships between classes to identify which objects to list. A single level of hierarchy has no reference to another level. A small amount of dev work would resolve this. The endpoint nodeshapes can be specified in the case of N=1 to not look for relationships to other classes. + - The higher limit of three is because the SHACL parsing is not completely recursive. It could be manually extended to N levels, however it would be better to write a general SHACL parsing library. diff --git a/README-OGC-Features.md b/README-OGC-Features.md index 59356108..bb5ca091 100644 --- a/README-OGC-Features.md +++ b/README-OGC-Features.md @@ -40,7 +40,6 @@ ex:BDRScientificNameQueryableShape . ``` It is recommended that templated SPARQL queries are used to periodically update the `sh:in` values, which correspond to enumerations. -# TODO other SHACL predicates can be reused to specify min/max values, etc. where the range is numeric and enumerations are not appropriate. When Prez starts, it will query the remote repository (typically a triplestore) for all Queryables. It queries for them using a CONSTRUCT query, serializes this as JSON-LD, and does a minimal transformation to produce the OGC Features compliant response. diff --git a/prez/static/endpoint_config.html b/prez/static/endpoint_config.html index 0169ba03..b7a95d95 100644 --- a/prez/static/endpoint_config.html +++ b/prez/static/endpoint_config.html @@ -31,13 +31,6 @@

Instructions:

-
- -
- -
-
-
@@ -54,11 +47,12 @@

Instructions:

const routeContainer = document.querySelector('div#route-container'); let routeCount = 0; - function addRemoveButton(element, container, removeFunction) { + function addRemoveButton(element, container, removeFunction, isDisabled = false) { const removeBtn = document.createElement('button'); removeBtn.className = 'button is-danger is-small'; removeBtn.textContent = 'Remove'; removeBtn.style.marginLeft = '10px'; + removeBtn.disabled = isDisabled; removeBtn.addEventListener('click', (e) => { e.preventDefault(); removeFunction(element, container); @@ -70,12 +64,48 @@

Instructions:

container.removeChild(element); } - document.querySelector('button.add-route').addEventListener('click', (e) => { - e.preventDefault(); - const newRoute = document.createElement('div'); - newRoute.className = 'box'; - newRoute.innerHTML = ` -

Route ${routeCount + 1}

+ function addRoute() { + if (routeCount === 0) { + const newRoute = document.createElement('div'); + newRoute.className = 'box'; + newRoute.innerHTML = ` +

Route ${routeCount + 1}

+
+ +
+ +
+
+
+ +
+ +
+
+
+
+ +
+
+
+ `; + routeContainer.appendChild(newRoute); + addRemoveButton(newRoute, routeContainer, removeElement, true); // Disable the remove button + routeCount++; + + const hierarchyRelationBtn = newRoute.querySelector('button.add-hierarchy-relation'); + hierarchyRelationBtn.addEventListener('click', addHierarchyRelation); + + // Automatically add the first hierarchy relation + addHierarchyRelation({ target: hierarchyRelationBtn }); + } + } + + function addHierarchyRelation(event) { + const newHierarchyRelation = document.createElement('div'); + newHierarchyRelation.className = 'box'; + newHierarchyRelation.innerHTML = ` +

Class Hierarchy

@@ -83,137 +113,115 @@

Route ${routeCount + 1}

-
- +
-
-
- -
+
+
+
-
`; - routeContainer.appendChild(newRoute); - addRemoveButton(newRoute, routeContainer, removeElement); - routeCount++; + const hierarchyRelationsContainer = event.target.closest('.box').querySelector('.hierarchy-relations-container'); + hierarchyRelationsContainer.appendChild(newHierarchyRelation); + addRemoveButton(newHierarchyRelation, hierarchyRelationsContainer, removeElement); - const hierarchyRelationBtn = newRoute.querySelector('button.add-hierarchy-relation'); - hierarchyRelationBtn.addEventListener('click', (e) => { - e.preventDefault(); - const newHierarchyRelation = document.createElement('div'); - newHierarchyRelation.className = 'box'; - newHierarchyRelation.innerHTML = ` -

Add Class Hierarchy

+ const classContainer = newHierarchyRelation.querySelector('.class-container'); + const relationContainer = newHierarchyRelation.querySelector('.relation-container'); + let classCount = 0; + + const classBtn = newHierarchyRelation.querySelector('button.add-class'); + classBtn.addEventListener('click', () => addClass(classContainer, relationContainer, classBtn)); + + // Automatically add two classes and one relation + addClass(classContainer, relationContainer, classBtn); + addClass(classContainer, relationContainer, classBtn); + } + + function addClass(classContainer, relationContainer, classBtn) { + const classCount = classContainer.children.length; + if (classCount < 3) { + const newClass = document.createElement('div'); + newClass.className = 'box class-item'; + newClass.innerHTML = ` +
Class at Hierarchy Level ${classCount + 1}
- +
- +
+
- +
-
-
-
-
`; - const hierarchyRelationsContainer = newRoute.querySelector(`div#hierarchy-relations-container-${routeCount - 1}`); - hierarchyRelationsContainer.appendChild(newHierarchyRelation); - addRemoveButton(newHierarchyRelation, hierarchyRelationsContainer, removeElement); - - const classContainer = newHierarchyRelation.querySelector(`div#class-container-${routeCount}`); - const relationContainer = newHierarchyRelation.querySelector(`div#relation-container-${routeCount}`); - let classCount = 0; - - const classBtn = newHierarchyRelation.querySelector('button.add-class'); - classBtn.addEventListener('click', (e) => { - e.preventDefault(); - if (classCount < 3) { - classCount++; - const newClass = document.createElement('div'); - newClass.className = 'box class-item'; - newClass.innerHTML = ` -
Class at Hierarchy Level ${classCount}
-
- -
- -
-
-
- -
- -
-
- `; - classContainer.appendChild(newClass); - addRemoveButton(newClass, classContainer, (element, container) => { - removeElement(element, container); - classCount--; - classBtn.disabled = false; - updateClassLevels(classContainer); - updateRelations(relationContainer, classCount); - }); - - if (classCount > 1) { - const newRelation = document.createElement('div'); - newRelation.className = 'box relation-item'; - newRelation.innerHTML = ` -
Relation between Classes at Hierarchy Levels ${classCount - 1} and ${classCount}
-
- -
-
- -
-
-
-
- -
- -
-
- `; - relationContainer.appendChild(newRelation); - addRemoveButton(newRelation, relationContainer, removeElement); - } - - if (classCount === 3) { - classBtn.disabled = true; - } + classContainer.appendChild(newClass); + addRemoveButton(newClass, classContainer, (element, container) => { + if (container.children.length > 2) { + removeElement(element, container); + updateClassLevels(classContainer); + updateRelations(relationContainer, classContainer.children.length); + classBtn.disabled = false; } }); - function updateClassLevels(container) { - const classes = container.querySelectorAll('.class-item'); - classes.forEach((classItem, index) => { - const levelTag = classItem.querySelector('.tag'); - levelTag.textContent = index + 1; - }); + if (classCount > 0) { + addRelation(relationContainer, classCount, classCount + 1); } - function updateRelations(container, classCount) { - const relations = container.querySelectorAll('.relation-item'); - relations.forEach((relationItem, index) => { - const title = relationItem.querySelector('h5'); - title.textContent = `Relation between Classes at Hierarchy Levels ${index + 1} and ${index + 2}`; - }); - while (relations.length > classCount - 1) { - container.removeChild(relations[relations.length - 1]); - } + if (classCount === 2) { + classBtn.disabled = true; } + } + } + + function addRelation(relationContainer, levelFrom, levelTo) { + const newRelation = document.createElement('div'); + newRelation.className = 'box relation-item'; + newRelation.innerHTML = ` +
Relation between Classes at Hierarchy Levels ${levelFrom} and ${levelTo}
+
+ +
+
+ +
+
+
+
+ +
+ +
+
+ `; + relationContainer.appendChild(newRelation); + } + + function updateClassLevels(container) { + const classes = container.querySelectorAll('.class-item'); + classes.forEach((classItem, index) => { + const levelTag = classItem.querySelector('.tag'); + levelTag.textContent = index + 1; }); - }); + } + + function updateRelations(container, classCount) { + const relations = container.querySelectorAll('.relation-item'); + relations.forEach((relationItem, index) => { + const title = relationItem.querySelector('h5'); + title.textContent = `Relation between Classes at Hierarchy Levels ${index + 1} and ${index + 2}`; + }); + while (relations.length > classCount - 1) { + container.removeChild(relations[relations.length - 1]); + } + } document.getElementById('configForm').addEventListener('submit', function(e) { e.preventDefault(); @@ -283,6 +291,9 @@
Relation between Classes at Hierarchy Levels ${classCount alert('An error occurred while submitting the configuration. Please try again.'); }); }); + + // Automatically add the first route when the page loads + window.addEventListener('load', addRoute); \ No newline at end of file From 10400634def5f2e40c3956d62dec43bade0bf3d9 Mon Sep 17 00:00:00 2001 From: Lawon Lewis Date: Mon, 7 Oct 2024 21:21:38 +1000 Subject: [PATCH 6/7] fix: relative path issue --- prez/app.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/prez/app.py b/prez/app.py index a37b84e2..46fca031 100755 --- a/prez/app.py +++ b/prez/app.py @@ -1,4 +1,5 @@ import logging +from pathlib import Path from contextlib import asynccontextmanager from functools import partial from textwrap import dedent @@ -184,7 +185,7 @@ def assemble_app( app.include_router(sparql_router) if _settings.configuration_mode: app.include_router(config_router) - app.mount("/static", StaticFiles(directory="static"), name="static") + app.mount("/static", StaticFiles(directory=Path(__file__).parent / "static"), name="static") if _settings.enable_ogc_features: app.mount( "/catalogs/{catalogId}/collections/{recordsCollectionId}/features", From de88713dc4bbaf5824c6c69923b4b203405c6a68 Mon Sep 17 00:00:00 2001 From: david Date: Tue, 8 Oct 2024 21:36:57 +1000 Subject: [PATCH 7/7] Pin alpine version. --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index f723f8bf..215a7019 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,13 @@ ARG PREZ_VERSION ARG PYTHON_VERSION=3.12.3 ARG POETRY_VERSION=1.8.3 +ARG ALPINE_VERSION=3.19 ARG VIRTUAL_ENV=/opt/venv # # Base # -FROM python:${PYTHON_VERSION}-alpine AS base +FROM python:${PYTHON_VERSION}-alpine${ALPINE_VERSION} AS base ARG POETRY_VERSION ARG VIRTUAL_ENV ENV VIRTUAL_ENV=${VIRTUAL_ENV} \ @@ -32,7 +33,7 @@ RUN ${VIRTUAL_ENV}/bin/pip3 install uvicorn # # Final # -FROM python:${PYTHON_VERSION}-alpine AS final +FROM python:${PYTHON_VERSION}-alpine${ALPINE_VERSION} AS final ARG PREZ_VERSION ENV PREZ_VERSION=${PREZ_VERSION}