David/dynamic endpoints

Dockerfile

@@ -1,12 +1,13 @@
 ARG PREZ_VERSION
 ARG PYTHON_VERSION=3.12.3
 ARG POETRY_VERSION=1.8.3
+ARG ALPINE_VERSION=3.19
 ARG VIRTUAL_ENV=/opt/venv
 
 #
 # Base
 #
-FROM python:${PYTHON_VERSION}-alpine AS base
+FROM python:${PYTHON_VERSION}-alpine${ALPINE_VERSION} AS base
 ARG POETRY_VERSION
 ARG VIRTUAL_ENV
 ENV VIRTUAL_ENV=${VIRTUAL_ENV} \
@@ -32,7 +33,7 @@ RUN ${VIRTUAL_ENV}/bin/pip3 install uvicorn
 #
 # Final
 #
-FROM python:${PYTHON_VERSION}-alpine AS final
+FROM python:${PYTHON_VERSION}-alpine${ALPINE_VERSION} AS final
 
 ARG PREZ_VERSION
 ENV PREZ_VERSION=${PREZ_VERSION}

README-Custom-Endpoints.md

@@ -0,0 +1,41 @@
+# Custom Data Endpoint Configuration
+
+Prez allows the configuration of custom endpoints.
+
+That is, you can specify the route structure e.g.
+
+`/catalogs/{catalogId}/products/{productId}`
+
+And then specify which classes these endpoints deliver, and what the RDF relations between those classes are.
+
+## Set up instructions
+To set up custom endpoints:
+1. Set the CONFIGURATION_MODE environment variable to "true"
+2. Go to the `/configure-endpoints` page and complete the form; submit
+3. Set the CUSTOM_ENDPOINTS environment variable to "true"
+4. Restart Prez. You should see the dynamic endpoints being created in the logs.
+5. Confirm your endpoints are working as expected.
+
+Once working as expected:
+1. Copy the "custom_endpoints.ttl" file from `prez/reference_data/data_endpoints_custom` to the remote backend (e.g. triplestore)
+    > Prez will preferentially use the custom endpoints specified in the triplestore over the ones specified in the local file.
+2. Set the CONFIGURATION_MODE environment variable to "false"
+
+## Limitations
+The following limitations apply at present:
+- The endpoint structure must be specified in the config to match what is input through the form.
+related to this:
+- Only one route can be specified (though multiple class hierarchies which use that one route can be specified)
+i.e. you can specify
+`/catalogs/{catalogId}/products/{productId}`
+but not
+`/catalogs/{catalogId}/products/{productId}`
+and
+`/datasets/{datasetId}/items/{itemsId}
+on the one prez instance.
+- This limitation is only due to link generation, which looks up the (currently) single endpoint structure variable in the config file.
+- This should be resolvable with a small amount of work. At link generation time, an endpoint nodeshape is in context, and endpoint nodeshapes are mapped to a route structure.
+
+- The number of hierarchy levels within a route must be two or three (i.e. 2 or 3 levels of classes = 4-6 listing/object endpoints)
+    - The lower limit of two is because prez uses the relationships between classes to identify which objects to list. A single level of hierarchy has no reference to another level. A small amount of dev work would resolve this. The endpoint nodeshapes can be specified in the case of N=1 to not look for relationships to other classes.
+    - The higher limit of three is because the SHACL parsing is not completely recursive. It could be manually extended to N levels, however it would be better to write a general SHACL parsing library.

README-OGC-Features.md

@@ -40,7 +40,6 @@ ex:BDRScientificNameQueryableShape
 .
 ```
 It is recommended that templated SPARQL queries are used to periodically update the `sh:in` values, which correspond to enumerations.
-# TODO other SHACL predicates can be reused to specify min/max values, etc. where the range is numeric and enumerations are not appropriate.
 
 When Prez starts, it will query the remote repository (typically a triplestore) for all Queryables.
 It queries for them using a CONSTRUCT query, serializes this as JSON-LD, and does a minimal transformation to produce the OGC Features compliant response. 

prez/app.py

@@ -1,4 +1,5 @@
 import logging
+from pathlib import Path
 from contextlib import asynccontextmanager
 from functools import partial
 from textwrap import dedent
@@ -8,6 +9,7 @@
 from fastapi.openapi.utils import get_openapi
 from rdflib import Graph
 from starlette.middleware.cors import CORSMiddleware
+from starlette.staticfiles import StaticFiles
 
 from prez.config import settings, Settings
 from prez.dependencies import (
@@ -29,10 +31,11 @@
     PrefixNotFoundException,
 )
 from prez.repositories import RemoteSparqlRepo, PyoxigraphRepo, OxrdflibRepo
+from prez.routers.custom_endpoints import create_dynamic_router
 from prez.routers.identifier import router as identifier_router
-from prez.routers.management import router as management_router
-from prez.routers.ogc_router import router as ogc_records_router
+from prez.routers.management import router as management_router, config_router
 from prez.routers.ogc_features_router import features_subapi
+from prez.routers.base_router import router as base_prez_router
 from prez.routers.sparql import router as sparql_router
 from prez.services.app_service import (
     healthcheck_sparql_endpoints,
@@ -113,7 +116,7 @@ async def lifespan(app: FastAPI):
     await prefix_initialisation(app.state.repo)
     await retrieve_remote_template_queries(app.state.repo)
     await create_profiles_graph(app.state.repo)
-    await create_endpoints_graph(app.state.repo)
+    await create_endpoints_graph(app.state)
     await count_objects(app.state.repo)
     await populate_api_info()
 
@@ -124,6 +127,9 @@ async def lifespan(app: FastAPI):
     await load_system_data_to_oxigraph(app.state.pyoxi_system_store)
     await load_annotations_data_to_oxigraph(app.state.annotations_store)
 
+    # dynamic routes are either: custom routes if enabled, else default prez "data" routes are added dynamically
+    app.include_router(create_dynamic_router())
+
     yield
 
     # Shutdown
@@ -175,14 +181,17 @@ def assemble_app(
     app.state.settings = _settings
 
     app.include_router(management_router)
-    app.include_router(ogc_records_router)
     if _settings.enable_sparql_endpoint:
         app.include_router(sparql_router)
+    if _settings.configuration_mode:
+        app.include_router(config_router)
+        app.mount("/static", StaticFiles(directory=Path(__file__).parent / "static"), name="static")
     if _settings.enable_ogc_features:
         app.mount(
             "/catalogs/{catalogId}/collections/{recordsCollectionId}/features",
             features_subapi,
         )
+    app.include_router(base_prez_router)
     app.include_router(identifier_router)
     app.openapi = partial(
         prez_open_api_metadata,

prez/config.py

@@ -80,6 +80,8 @@ class Settings(BaseSettings):
     ]
     enable_sparql_endpoint: bool = False
     enable_ogc_features: bool = True
+    custom_endpoints: bool = False
+    configuration_mode: bool = False
     temporal_predicate: Optional[URIRef] = SDO.temporal
     endpoint_to_template_query_filename: Optional[Dict[str, str]] = {}
 

prez/models/endpoint_config.py

@@ -0,0 +1,99 @@
+from typing import List, Literal
+
+from pydantic import BaseModel, field_validator
+from rdflib import URIRef
+
+
+class HierarchyItem(BaseModel):
+    rdfClass: str
+    className: str
+    hierarchyLevel: int
+
+    @classmethod
+    @field_validator("rdfClass")
+    def validate_uri(cls, v: str) -> str:
+        try:
+            URIRef(v)
+            return v
+        except ValueError as e:
+            raise ValueError(f"Invalid URI: {v}. Error: {str(e)}")
+
+
+class Relation(BaseModel):
+    levelFrom: int
+    levelTo: int
+    direction: Literal["outbound", "inbound"]
+    rdfPredicate: str
+
+    @classmethod
+    @field_validator("rdfPredicate")
+    def validate_uri(cls, v: str) -> str:
+        try:
+            URIRef(v)
+            return v
+        except ValueError as e:
+            raise ValueError(f"Invalid URI: {v}. Error: {str(e)}")
+
+
+class HierarchyRelation(BaseModel):
+    name: str
+    hierarchy: List[HierarchyItem]
+    relations: List[Relation]
+
+    @classmethod
+    @field_validator("relations")
+    def validate_relations_count(
+        cls, v: List[Relation], values: dict
+    ) -> List[Relation]:
+        hierarchy = values.get("hierarchy")
+        if hierarchy and len(v) != len(hierarchy) - 1:
+            raise ValueError(
+                f"Number of relations ({len(v)}) should be one less than the number of hierarchy items ({len(hierarchy)})"
+            )
+        return v
+
+
+class Route(BaseModel):
+    name: str
+    fullApiPath: str
+    hierarchiesRelations: List[HierarchyRelation]
+
+
+class RootModel(BaseModel):
+    routes: List[Route]
+
+
+configure_endpoings_example = {
+    "configName": "Prez Example Endpoint Configuration",
+    "routes": [
+        {
+            "name": "Single hierarchy with Datasets within Catalogs",
+            "fullApiPath": "/catalogs/{catalogId}/datasets/{datasetId}",
+            "hierarchiesRelations": [
+                {
+                    "name": "Catalogue of Datasets",
+                    "hierarchy": [
+                        {
+                            "rdfClass": "http://www.w3.org/ns/dcat#Catalog",
+                            "className": "Catalog",
+                            "hierarchyLevel": 1,
+                        },
+                        {
+                            "rdfClass": "http://www.w3.org/ns/dcat#Dataset",
+                            "className": "DCAT Dataset",
+                            "hierarchyLevel": 2,
+                        },
+                    ],
+                    "relations": [
+                        {
+                            "levelFrom": 1,
+                            "levelTo": 2,
+                            "direction": "outbound",
+                            "rdfPredicate": "http://purl.org/dc/terms/hasPart",
+                        }
+                    ],
+                }
+            ],
+        }
+    ],
+}

prez/reference_data/endpoints/endpoint_metadata.ttl → prez/reference_data/endpoints/base/endpoint_metadata.ttl

@@ -29,36 +29,6 @@ ogce:cql-post
     ont:relevantShapes ex:CQL ;
     .
 
-ogce:catalog-listing
-    a ont:ListingEndpoint ;
-    ont:relevantShapes ex:Catalogs ;
-.
-
-ogce:catalog-object
-    a ont:ObjectEndpoint ;
-    ont:relevantShapes ex:Catalogs ;
-.
-
-ogce:collection-listing
-    a ont:ListingEndpoint ;
-    ont:relevantShapes ex:Collections ;
-.
-
-ogce:collection-object
-    a ont:ObjectEndpoint ;
-    ont:relevantShapes ex:Collections ;
-.
-
-ogce:item-listing
-    a ont:ListingEndpoint ;
-    ont:relevantShapes ex:Feature , ex:ConceptSchemeConcept , ex:CollectionConcept , ex:Resource ;
-.
-
-ogce:item-object
-    a ont:ObjectEndpoint ;
-    ont:relevantShapes ex:Feature , ex:ConceptSchemeConcept , ex:CollectionConcept , ex:Resource ;
-.
-
 ogce:search
     a ont:ListingEndpoint ;
     ont:relevantShapes ex:Search ;

prez/reference_data/endpoints/endpoint_nodeshapes.ttl → prez/reference_data/endpoints/base/endpoint_nodeshapes.ttl

@@ -12,65 +12,6 @@
 @prefix skos: <http://www.w3.org/2004/02/skos/core#> .
 @prefix altr-ext: <http://www.w3.org/ns/dx/connegp/altr-ext#> .
 
-ex:Catalogs
-    a sh:NodeShape ;
-    ont:hierarchyLevel 1 ;
-    sh:targetClass dcat:Catalog ;
-    sh:property [
-        sh:path dcterms:hasPart ;
-        sh:or (
-            [ sh:class skos:ConceptScheme ]
-            [ sh:class skos:Collection ]
-            [ sh:class dcat:Dataset ]
-        ) ;
-    ] .
-
-ex:Collections
-    a sh:NodeShape ;
-    ont:hierarchyLevel 2 ;
-    sh:targetClass skos:ConceptScheme , skos:Collection , dcat:Dataset ;
-    sh:property [
-        sh:path [ sh:inversePath dcterms:hasPart ] ;
-        sh:class dcat:Catalog ;
-    ] .
-
-
-ex:ConceptSchemeConcept
-    a sh:NodeShape ;
-    ont:hierarchyLevel 3 ;
-    sh:targetClass skos:Concept ;
-    sh:property [
-        sh:path skos:inScheme ;
-        sh:class skos:ConceptScheme ;
-    ] , [
-        sh:path ( skos:inScheme [ sh:inversePath dcterms:hasPart ] );
-        sh:class dcat:Catalog ;
-    ] .
-
-ex:CollectionConcept
-    a sh:NodeShape ;
-    ont:hierarchyLevel 3 ;
-    sh:targetClass skos:Concept ;
-    sh:property [
-        sh:path [ sh:inversePath skos:member ] ;
-        sh:class skos:Collection ;
-    ] , [
-        sh:path ( [ sh:inversePath skos:member ] [ sh:inversePath dcterms:hasPart ] );
-        sh:class dcat:Catalog ;
-    ] .
-
-ex:Resource
-    a sh:NodeShape ;
-    ont:hierarchyLevel 3 ;
-    sh:targetClass geo:FeatureCollection ;
-    sh:property [
-        sh:path [ sh:inversePath dcterms:hasPart ] ;
-        sh:class dcat:Dataset ;
-    ] , [
-        sh:path ( [ sh:inversePath dcterms:hasPart ] [ sh:inversePath dcterms:hasPart ] );
-        sh:class dcat:Catalog ;
-    ] .
-
 ex:Profiles
     a sh:NodeShape ;
     ont:hierarchyLevel 1 ;

prez/reference_data/endpoints/data_endpoints_custom/example_custom_endpoints.ttl

@@ -0,0 +1,43 @@
+@prefix ex: <http://example.org/> .
+@prefix ont: <https://prez.dev/ont/> .
+@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
+@prefix schema: <https://schema.org/> .
+@prefix sh: <http://www.w3.org/ns/shacl#> .
+@prefix skos: <http://www.w3.org/2004/02/skos/core#> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+
+ex:catalogs-listing a ont:ListingEndpoint , ont:DynamicEndpoint ;
+    rdfs:label "Catalogs Listing" ;
+    ont:apiPath "/catalogs" ;
+    ont:relevantShapes ex:shape-R0-HL1 .
+
+ex:catalogs-object a ont:ObjectEndpoint , ont:DynamicEndpoint ;
+    rdfs:label "Catalogs Object" ;
+    ont:apiPath "/catalogs/{catalogId}" ;
+    ont:relevantShapes ex:shape-R0-HL1 .
+
+ex:items-listing a ont:ListingEndpoint , ont:DynamicEndpoint ;
+    rdfs:label "Items Listing" ;
+    ont:apiPath "/catalogs/{catalogId}/items" ;
+    ont:relevantShapes ex:shape-R0-HL2 .
+
+ex:items-object a ont:ObjectEndpoint , ont:DynamicEndpoint ;
+    rdfs:label "Items Object" ;
+    ont:apiPath "/catalogs/{catalogId}/items/{itemId}" ;
+    ont:relevantShapes ex:shape-R0-HL2 .
+
+ex:shape-R0-HL1 a sh:NodeShape ;
+    sh:property [ sh:or ( [ sh:class <https://linked.data.gov.au/def/nsl/Taxon> ] [ sh:class schema:CreativeWork ] [ sh:class <https://linked.data.gov.au/def/nsl/Usage> ] [ sh:class <https://linked.data.gov.au/def/nsl/TaxonName> ] ) ;
+            sh:path skos:member ] ;
+    sh:targetClass skos:Collection ;
+    ont:hierarchyLevel 1 .
+
+ex:shape-R0-HL2 a sh:NodeShape ;
+    sh:property [ sh:class skos:Collection ;
+            sh:path [ sh:inversePath skos:member ] ] ;
+    sh:targetClass <https://linked.data.gov.au/def/nsl/Taxon>,
+        <https://linked.data.gov.au/def/nsl/TaxonName>,
+        <https://linked.data.gov.au/def/nsl/Usage>,
+        schema:CreativeWork ;
+    ont:hierarchyLevel 2 .