diff --git a/backend/collect/__init__.py b/backend/collect/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/collect/api.py b/backend/collect/api.py new file mode 100644 index 00000000..ed105dd0 --- /dev/null +++ b/backend/collect/api.py @@ -0,0 +1,43 @@ +from rest_framework.viewsets import ModelViewSet +from rest_framework.exceptions import NotFound +from rdflib import URIRef, RDF, Graph +from django.conf import settings + +from projects.api import user_projects +from collect.rdf_models import EDPOPCollection +from collect.utils import collection_exists, collection_graph +from triplestore.constants import EDPOPCOL +from collect.serializers import CollectionSerializer +from collect.permissions import CollectionPermission + +class CollectionViewSet(ModelViewSet): + ''' + Viewset for listing or retrieving collections + ''' + + lookup_value_regex = '.+' + serializer_class = CollectionSerializer + permission_classes = [CollectionPermission] + + def get_queryset(self): + projects = user_projects(self.request.user) + return [ + EDPOPCollection(collection_graph(uri), uri) + for project in projects + for uri in project.rdf_model().collections + ] + + + def get_object(self): + uri = URIRef(self.kwargs['pk']) + + if not collection_exists(uri): + raise NotFound(f'Collection does not exist') + + store = settings.RDFLIB_STORE + context = next(store.contexts((uri, RDF.type, EDPOPCOL.Collection))) + graph = Graph(store, context) + collection = EDPOPCollection(graph, uri) + self.check_object_permissions(self.request, collection) + return collection + diff --git a/backend/collect/api_test.py b/backend/collect/api_test.py new file mode 100644 index 00000000..afe24524 --- /dev/null +++ b/backend/collect/api_test.py @@ -0,0 +1,129 @@ +from django.test import Client +from rest_framework.status import is_success, is_client_error +from rdflib import URIRef, RDF, Literal +from django.conf import settings +from urllib.parse import quote +from typing import Dict + +from triplestore.constants import EDPOPCOL, AS +from collect.utils import collection_uri +from projects.models import Project + +def example_collection_data(project_name) -> Dict: + return { + 'name': 'My collection', + 'summary': 'These are my favourite records', + 'project': project_name, + } + +def post_collection(client, project_name): + data = example_collection_data(project_name) + return client.post('/api/collections/', data, content_type='application/json') + +def test_create_collection(db, user, project, client: Client): + client.force_login(user) + + response = post_collection(client, project.name) + assert is_success(response.status_code) + uri = URIRef(response.data['uri']) + + store = settings.RDFLIB_STORE + assert any(store.triples((uri, RDF.type, EDPOPCOL.Collection))) + + +def test_create_fails_if_collection_exists(db, user, project, client: Client): + client.force_login(user) + success_response = post_collection(client, project.name) + assert is_success(success_response.status_code) + uri = URIRef(success_response.data['uri']) + + # try to create a collection at the same location + fail_response = client.post('/api/collections/', { + 'name': 'My collection', + 'summary': 'I like these too', + 'project': project.name + }) + assert is_client_error(fail_response.status_code) + + store = settings.RDFLIB_STORE + is_stored = lambda triple: any(store.triples(triple)) + assert is_stored((uri, AS.summary, Literal('These are my favourite records'))) + assert not is_stored((uri, AS.summary, Literal('I like these too'))) + + +def test_list_collections(db, user, project, client: Client): + client.force_login(user) + + response = client.get('/api/collections/') + assert is_success(response.status_code) + assert len(response.data) == 0 + + response = post_collection(client, project.name) + + response = client.get('/api/collections/') + assert is_success(response.status_code) + assert len(response.data) == 1 + assert response.data[0]['uri'] == settings.RDF_NAMESPACE_ROOT + 'collections/my_collection' + assert response.data[0]['name'] == 'My collection' + + +def collection_detail_url(collection_uri: str) -> str: + return '/api/collections/{}/'.format(quote(collection_uri, safe='')) + + +def test_retrieve_collection(db, user, project, client: Client): + client.force_login(user) + create_response = post_collection(client, project.name) + + + correct_url = collection_detail_url(create_response.data['uri']) + nonexistent_uri = collection_uri('does not exist') + + not_found_response = client.get(collection_detail_url(nonexistent_uri)) + assert not_found_response.status_code == 404 + + success_response = client.get(correct_url) + assert is_success(success_response.status_code) + assert success_response.data['name'] == 'My collection' + + client.logout() + no_permission_response = client.get(correct_url) + assert no_permission_response.status_code == 403 + +def test_delete_collection(db, user, project, client: Client): + client.force_login(user) + create_response = post_collection(client, project.name) + + detail_url = collection_detail_url(create_response.data['uri']) + delete_response = client.delete(detail_url) + assert is_success(delete_response.status_code) + + retrieve_response = client.get(detail_url) + assert retrieve_response.status_code == 404 + +def test_update_collection(db, user, project, client: Client): + client.force_login(user) + + create_response = post_collection(client, project.name) + detail_url = collection_detail_url(create_response.data['uri']) + + data = example_collection_data(project.name) + data.update({'summary': 'I don\'t like these anymore'}) + + update_response = client.put(detail_url, data, content_type='application/json') + assert is_success(update_response.status_code) + assert update_response.data['summary'] == 'I don\'t like these anymore' + + +def test_project_validation(db, user, client: Client): + client.force_login(user) + + Project.objects.create(name='secret', display_name='Top secret records') + + response = client.post('/api/collections/', { + 'name': 'new collection', + 'summary': None, + 'project': 'secret', + }, content_type='application/json') + + assert is_client_error(response.status_code) diff --git a/backend/collect/apps.py b/backend/collect/apps.py new file mode 100644 index 00000000..36b7babb --- /dev/null +++ b/backend/collect/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class CollectConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'collect' diff --git a/backend/collect/conftest.py b/backend/collect/conftest.py new file mode 100644 index 00000000..f8082c5b --- /dev/null +++ b/backend/collect/conftest.py @@ -0,0 +1,20 @@ +import pytest +from django.contrib.auth.models import User +from projects.models import Project + + +@pytest.fixture() +def user(db) -> User: + return User.objects.create( + username='tester', + password='secret' + ) + +@pytest.fixture() +def project(db, user): + project = Project.objects.create( + name='test_project', + display_name='Test project' + ) + project.users.add(user) + return project diff --git a/backend/collect/migrations/__init__.py b/backend/collect/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/collect/permissions.py b/backend/collect/permissions.py new file mode 100644 index 00000000..14f6f24f --- /dev/null +++ b/backend/collect/permissions.py @@ -0,0 +1,17 @@ +from rest_framework import permissions + +from projects.models import Project + +class CollectionPermission(permissions.BasePermission): + ''' + Checks whether the user has access to read or write a collection. + ''' + + def has_object_permission(self, request, view, obj): + project_uri = obj.project + project = Project.objects.get(uri=project_uri) + + if request.method in permissions.SAFE_METHODS: + return project.permit_query_by(request.user) + else: + return project.permit_update_by(request.user) diff --git a/backend/collect/rdf_models.py b/backend/collect/rdf_models.py new file mode 100644 index 00000000..05b11aed --- /dev/null +++ b/backend/collect/rdf_models.py @@ -0,0 +1,39 @@ +from rdflib import RDFS, IdentifiedNode, URIRef +from typing import Iterable + +from triplestore.utils import Triples +from triplestore.constants import EDPOPCOL, AS +from triplestore.rdf_model import RDFModel +from triplestore.rdf_field import RDFField, RDFUniquePropertyField + + +class CollectionMembersField(RDFField): + def get(self, instance: RDFModel): + return [ + s + for (s, p, o) in self._stored_triples(instance) + ] + + + def _stored_triples(self,instance: RDFModel) -> Triples: + g = self.get_graph(instance) + return g.triples((None, RDFS.member, instance.uri)) + + + def _triples_to_store(self, instance: RDFModel, value: Iterable[IdentifiedNode]) -> Triples: + return [ + (uri, RDFS.member, instance.uri) + for uri in value + ] + + +class EDPOPCollection(RDFModel): + ''' + RDF model for EDPOP collections. + ''' + rdf_class = EDPOPCOL.Collection + + name = RDFUniquePropertyField(AS.name) + summary = RDFUniquePropertyField(AS.summary) + project = RDFUniquePropertyField(AS.context) + records = CollectionMembersField() diff --git a/backend/collect/rdf_models_test.py b/backend/collect/rdf_models_test.py new file mode 100644 index 00000000..b758937c --- /dev/null +++ b/backend/collect/rdf_models_test.py @@ -0,0 +1,38 @@ +import pytest +from rdflib import URIRef, RDF, RDFS +from django.conf import settings + +from triplestore.constants import AS, EDPOPCOL +from projects.models import Project +from projects.rdf_models import RDFProject +from collect.rdf_models import EDPOPCollection + +@pytest.fixture() +def project(db): + project = Project.objects.create(name='test', display_name='Test') + rdf_project = RDFProject(project.graph(), project.identifier()) + return rdf_project + +def test_collection_model(project): + uri = URIRef('test-collection', base='https://test.org/collections/') + + collection = EDPOPCollection(project.graph, uri) + collection.name = 'Test collection' + collection.project = project.uri + collection.records = [ + URIRef('https://example.org/example1'), + URIRef('https://example.org/example2') + ] + collection.save() + + store = settings.RDFLIB_STORE + + assert any(store.triples((collection.uri, RDF.type, EDPOPCOL.Collection))) + assert any(store.triples((collection.uri, AS.context, project.uri))) + assert any(store.triples((None, RDFS.member, collection.uri))) + + collection.delete() + + assert not any(store.triples((collection.uri, RDF.type, EDPOPCOL.Collection))) + assert not any(store.triples((collection.uri, AS.context, project.uri))) + assert not any(store.triples((None, RDFS.member, collection.uri))) diff --git a/backend/collect/serializers.py b/backend/collect/serializers.py new file mode 100644 index 00000000..a37f436b --- /dev/null +++ b/backend/collect/serializers.py @@ -0,0 +1,74 @@ +from rest_framework import serializers +from rdflib import URIRef + +from collect.rdf_models import EDPOPCollection +from collect.utils import collection_uri, collection_exists, collection_graph +from projects.models import Project + + +class ProjectField(serializers.Field): + def __init__(self, **kwargs): + super().__init__( **kwargs) + + def to_internal_value(self, data): + project = Project.objects.get(name=data) + return URIRef(project.uri) + + def to_representation(self, value): + project = Project.objects.get(uri=str(value)) + return project.name + + +def can_update_project(data): + ''' + Validates that the specified project is one the user is allowed to write to. + + Note: not to be confused with CollectionPermission. That permission checks whether the + user has access to a collection its current context; this validator checks the + user-submitted data. This prevents users from adding collections to projects they + cannot access. + ''' + + project_uri = data['project'] + user = data['user'] + + project_obj = Project.objects.get(uri=str(project_uri)) + if not project_obj.permit_update_by(user): + raise serializers.ValidationError( + 'No permission to write to this project' + ) + + +class CollectionSerializer(serializers.Serializer): + name = serializers.CharField(max_length=128) + summary = serializers.CharField( + max_length=1024, required=False, allow_null=True, default=None + ) + project = ProjectField() + uri = serializers.URLField(read_only=True) + user = serializers.HiddenField(default=serializers.CurrentUserDefault()) + + class Meta: + validators = [can_update_project] + + def create(self, validated_data): + project_uri = validated_data['project'] + uri = collection_uri(validated_data['name']) + graph = collection_graph(uri) + + if collection_exists(uri): + raise serializers.ValidationError(f'Collection {uri} already exists') + + collection = EDPOPCollection(graph, uri) + collection.name = validated_data['name'] + collection.summary = validated_data['summary'] + collection.project = project_uri + collection.save() + return collection + + def update(self, instance: EDPOPCollection, validated_data): + instance.name = validated_data['name'] + instance.summary = validated_data['summary'] + instance.project = validated_data['project'] + instance.save() + return instance diff --git a/backend/collect/utils.py b/backend/collect/utils.py new file mode 100644 index 00000000..69998399 --- /dev/null +++ b/backend/collect/utils.py @@ -0,0 +1,28 @@ +from django.conf import settings +from rdflib import RDF, URIRef, Graph +import re + +from triplestore.constants import EDPOPCOL + +def _name_to_slug(name: str) -> str: + lowered = name.lower() + cleaned = re.sub(r'[^a-z0-9\-_\s]', '', lowered) + stripped = re.sub(r'(\W+$|^\W+)', '', cleaned) + no_spaces = re.sub(r'\s+', '_', stripped) + return no_spaces + + +def collection_uri(name: str): + id = _name_to_slug(name) + return URIRef(settings.RDF_NAMESPACE_ROOT + 'collections/' + id) + + +def collection_exists(uri: URIRef): + store = settings.RDFLIB_STORE + triples = store.triples((uri, RDF.type, EDPOPCOL.Collection)) + return any(triples) + + +def collection_graph(uri: URIRef): + store = settings.RDFLIB_STORE + return Graph(store=store, identifier=uri) \ No newline at end of file diff --git a/backend/collect/utils_test.py b/backend/collect/utils_test.py new file mode 100644 index 00000000..4609c21b --- /dev/null +++ b/backend/collect/utils_test.py @@ -0,0 +1,16 @@ +from collect.utils import _name_to_slug + + +def test_name_to_slug(): + test_cases = [ + ('Test!!', 'test'), + ('Test test test', 'test_test_test'), + ('test/test?test=test&testing=true', 'testtesttesttesttestingtrue'), + ('https://example.com', 'httpsexamplecom'), + ('Alice\'s favourite books 📚️😊', 'alices_favourite_books'), + ('*?!##)', ''), + ('Test\n\n\tTest\n', 'test_test'), + ] + + for value, expected in test_cases: + assert _name_to_slug(value) == expected diff --git a/backend/edpop/urls.py b/backend/edpop/urls.py index 9ab1228b..aad6dabc 100644 --- a/backend/edpop/urls.py +++ b/backend/edpop/urls.py @@ -15,12 +15,30 @@ """ from django.urls import include, path from django.contrib import admin +from rest_framework import routers +from django.contrib.staticfiles.urls import staticfiles_urlpatterns +from vre.api import RecordViewSet, AnnotationViewSet, SearchViewSet, AddRecordsViewSet +from collect.api import CollectionViewSet + +api_router = routers.DefaultRouter() +api_router.register(r'records', RecordViewSet) +api_router.register(r'annotations', AnnotationViewSet) +api_router.register(r'search', SearchViewSet, basename='search') +api_router.register(r'add-selection', + AddRecordsViewSet, + basename='add-selection') +api_router.register('collections', CollectionViewSet, basename='collections') urlpatterns = [ path('admin/', admin.site.urls), + path('api-auth/', + include('rest_framework.urls', namespace='rest_framework')), + path('api/', include(api_router.urls)), path('', include('catalogs.urls')), path('', include('accounts.urls')), path('', include('projects.urls')), path('', include('vre.urls')), ] + +urlpatterns += staticfiles_urlpatterns() diff --git a/backend/projects/migrations/0002_researchgroups_to_projects.py b/backend/projects/migrations/0002_researchgroups_to_projects.py index 5616fef0..45aa69b2 100644 --- a/backend/projects/migrations/0002_researchgroups_to_projects.py +++ b/backend/projects/migrations/0002_researchgroups_to_projects.py @@ -2,7 +2,7 @@ from django.db import migrations from projects.migration_utils import name_to_slug -from projects.signals import store_project_graph, delete_project_graph +from projects.signals import store_project_graph, delete_project_graph, set_project_uri from projects.models import Project as ProjectCurrent def research_groups_to_projects(apps, schema_editor): @@ -30,6 +30,7 @@ def research_groups_to_projects(apps, schema_editor): summary=project.summary ) + set_project_uri(ProjectCurrent, project_with_rdf_methods) store_project_graph(ProjectCurrent, project_with_rdf_methods, created=False) @@ -53,6 +54,7 @@ def projects_to_research_groups(apps, schema_editor): summary=project.summary ) + set_project_uri(ProjectCurrent, project_with_rdf_methods) delete_project_graph(ProjectCurrent, project_with_rdf_methods) class Migration(migrations.Migration): diff --git a/backend/projects/migrations/0003_project_uri.py b/backend/projects/migrations/0003_project_uri.py new file mode 100644 index 00000000..354baefc --- /dev/null +++ b/backend/projects/migrations/0003_project_uri.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.13 on 2024-07-02 13:00 + +from django.db import migrations, models + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0002_researchgroups_to_projects'), + ] + + operations = [ + migrations.AddField( + model_name='project', + name='uri', + field=models.CharField(blank=True, help_text='URI for the project in RDF data', max_length=256), + ), + ] diff --git a/backend/projects/migrations/0004_fill_project_uri.py b/backend/projects/migrations/0004_fill_project_uri.py new file mode 100644 index 00000000..3459297c --- /dev/null +++ b/backend/projects/migrations/0004_fill_project_uri.py @@ -0,0 +1,28 @@ +# Generated by Django 4.2.13 on 2024-07-02 13:01 + +from django.db import migrations +from projects.signals import set_project_uri + +def fill_project_uri(apps, schema_editor): + Project = apps.get_model('projects', 'Project') + + for project in Project.objects.all(): + set_project_uri(Project, project) + project.save() + +def clear_project_uri(apps, schema_editor): + Project = apps.get_model('projects', 'Project') + + for project in Project.objects.all(): + project.uri = '' + project.save() + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0003_project_uri'), + ] + + operations = [ + migrations.RunPython(fill_project_uri, reverse_code=clear_project_uri) + ] diff --git a/backend/projects/migrations/0005_alter_project_uri.py b/backend/projects/migrations/0005_alter_project_uri.py new file mode 100644 index 00000000..8e78dd45 --- /dev/null +++ b/backend/projects/migrations/0005_alter_project_uri.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.13 on 2024-07-02 13:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0004_fill_project_uri'), + ] + + operations = [ + migrations.AlterField( + model_name='project', + name='uri', + field=models.CharField(help_text='URI for the project in RDF data', max_length=256, unique=True), + ), + ] diff --git a/backend/projects/models.py b/backend/projects/models.py index d72deaa1..34a21d68 100644 --- a/backend/projects/models.py +++ b/backend/projects/models.py @@ -4,6 +4,8 @@ from rdflib import URIRef, Graph from django.conf import settings +from projects.rdf_models import RDFProject + class Project(models.Model): ''' @@ -18,6 +20,11 @@ class Project(models.Model): unique=True, help_text='Identifier of the project; used in IRIs for the project\'s RDF data', ) + uri = models.CharField( + max_length=256, + unique=True, + help_text='URI for the project in RDF data', + ) display_name = models.CharField( max_length=256, help_text='Human-friendly name for the project', @@ -71,8 +78,8 @@ def identifier(self) -> URIRef: This is a node within the project graph; it can be used to give context to the project. ''' - if self.name: - return URIRef(self.name, base=self._graph_identifier()) + if self.uri: + return URIRef(self.uri) def permit_query_by(self, user: User) -> bool: @@ -97,3 +104,6 @@ def _granted_access(self, user: User) -> bool: if user.is_anonymous: return False return self.users.contains(user) or self.groups.filter(user=user).exists() + + def rdf_model(self): + return RDFProject(self.graph(), self.identifier()) diff --git a/backend/projects/rdf_models.py b/backend/projects/rdf_models.py index 1ed68e61..633d2515 100644 --- a/backend/projects/rdf_models.py +++ b/backend/projects/rdf_models.py @@ -1,9 +1,56 @@ +from rdflib import Graph, IdentifiedNode +from typing import Iterable +from django.conf import settings + from triplestore.constants import AS, EDPOPCOL from triplestore.rdf_model import RDFModel -from triplestore.rdf_field import RDFUniquePropertyField +from triplestore.rdf_field import RDFUniquePropertyField, RDFQuadField +from triplestore.utils import Quads + + +class CollectionsField(RDFQuadField): + ''' + Field containing the collections within a project. + + Collections are linked to projects through `as:context`. The context of a collection + is stored in the graph of the collection, not the graph of the project. + ''' + + def get(self, instance: RDFModel): + return [ + s + for (s, p, o, g) in self._stored_quads(instance) + ] + + def _stored_quads(self, instance: RDFModel) -> Quads: + store = settings.RDFLIB_STORE + results = store.query(f''' + SELECT ?col WHERE {{ + ?col a edpopcol:Collection ; + as:context <{instance.uri}> . + }} + ''', initNs={'as': AS, 'edpopcol': EDPOPCOL}) + + return [ + (result, AS.context, instance.uri, Graph(store, result)) + for (result, ) in results + ] + + def _quads_to_store(self, instance: RDFModel, value: Iterable[IdentifiedNode]) -> Quads: + return [ + (uri, AS.context, instance.uri, uri) + for uri in value + ] + class RDFProject(RDFModel): + ''' + RDF representation of a project. + ''' + rdf_class = EDPOPCOL.Project name = RDFUniquePropertyField(AS.name) summary = RDFUniquePropertyField(AS.summary) + + collections = CollectionsField() diff --git a/backend/projects/rdf_models_test.py b/backend/projects/rdf_models_test.py new file mode 100644 index 00000000..eb65a471 --- /dev/null +++ b/backend/projects/rdf_models_test.py @@ -0,0 +1,27 @@ +from rdflib import URIRef, RDF +from django.conf import settings + +from triplestore.constants import AS, EDPOPCOL +from projects.models import Project +from projects.rdf_models import RDFProject + +def test_project_collections(db): + store = settings.RDFLIB_STORE + sql_project = Project.objects.create(name='test', display_name='Test') + + g = sql_project.graph() + uri = sql_project.identifier() + + project = RDFProject(g, uri) + + assert project.collections == [] + + collection = URIRef('https://test.org/collections/1') + + g.add((collection, RDF.type, EDPOPCOL.Collection)) + g.add((collection, AS.context, project.uri)) + store.commit() + + project.refresh_from_store() + + assert project.collections == [collection] diff --git a/backend/projects/signals.py b/backend/projects/signals.py index 92b03f00..a29864b6 100644 --- a/backend/projects/signals.py +++ b/backend/projects/signals.py @@ -1,9 +1,18 @@ -from django.db.models.signals import post_save, post_delete +from django.db.models.signals import pre_save, post_save, post_delete from django.dispatch import receiver +from django.conf import settings from projects.models import Project from projects.rdf_models import RDFProject +@receiver(pre_save, sender=Project) +def set_project_uri(sender, instance: Project, **kwargs): + ''' + Set project URI if it is empty. + ''' + if not instance.uri: + instance.uri = settings.RDF_NAMESPACE_ROOT + 'projects/' + instance.name + @receiver(post_save, sender=Project) def store_project_graph(sender, instance: Project, created, **kwargs): ''' diff --git a/backend/vre/urls.py b/backend/vre/urls.py index bad9150d..66d0a215 100644 --- a/backend/vre/urls.py +++ b/backend/vre/urls.py @@ -1,25 +1,8 @@ -from django.urls import path, include, re_path -from django.contrib.staticfiles.urls import staticfiles_urlpatterns +from django.urls import re_path -from rest_framework import routers - -from . import views, api - -api_router = routers.DefaultRouter() -api_router.register(r'collections', api.CollectionViewSet) -api_router.register(r'records', api.RecordViewSet) -api_router.register(r'annotations', api.AnnotationViewSet) -api_router.register(r'search', api.SearchViewSet, basename='search') -api_router.register(r'add-selection', - api.AddRecordsViewSet, - basename='add-selection') +from . import views urlpatterns = [ - path('api/', include(api_router.urls)), - path('api-auth/', - include('rest_framework.urls', namespace='rest_framework')), - path('', views.index, name='index'), re_path(r".*", views.index, name='index'), ] -urlpatterns += staticfiles_urlpatterns()