Merge pull request #2082 from dandi/upload-lifecycle-management

Add incomplete upload dialog to DLP when unembargo is blocked

dandiapi/api/tests/test_dandiset.py

@@ -1116,3 +1116,86 @@ def test_dandiset_contact_person_malformed_contributors(api_client, draft_versio
     )
 
     assert results.data['draft_version']['dandiset']['contact_person'] == ''
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize('embargoed', [False, True])
+def test_dandiset_rest_list_active_uploads_not_owner(api_client, user, dandiset_factory, embargoed):
+    ds = dandiset_factory(
+        embargo_status=Dandiset.EmbargoStatus.EMBARGOED
+        if embargoed
+        else Dandiset.EmbargoStatus.OPEN
+    )
+
+    # Test unauthenticated
+    response = api_client.get(f'/api/dandisets/{ds.identifier}/uploads/')
+    assert response.status_code == 401
+
+    # Test unauthorized
+    api_client.force_authenticate(user=user)
+    response = api_client.get(f'/api/dandisets/{ds.identifier}/uploads/')
+    assert response.status_code == 403
+
+
+@pytest.mark.django_db
+def test_dandiset_rest_list_active_uploads(
+    authenticated_api_client, user, draft_version, upload_factory
+):
+    ds = draft_version.dandiset
+
+    assign_perm('owner', user, ds)
+    upload = upload_factory(dandiset=ds)
+
+    response = authenticated_api_client.get(f'/api/dandisets/{ds.identifier}/uploads/')
+    assert response.status_code == 200
+    data = response.json()
+    assert data['count'] == 1
+    assert len(data['results']) == 1
+    assert data['results'][0]['upload_id'] == upload.upload_id
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize('embargoed', [False, True])
+def test_dandiset_rest_clear_active_uploads_not_owner(
+    api_client, user, dandiset_factory, upload_factory, embargoed
+):
+    ds = dandiset_factory(
+        embargo_status=Dandiset.EmbargoStatus.EMBARGOED
+        if embargoed
+        else Dandiset.EmbargoStatus.OPEN
+    )
+
+    upload_factory(dandiset=ds)
+
+    # Test unauthenticated
+    response = api_client.delete(f'/api/dandisets/{ds.identifier}/uploads/')
+    assert response.status_code == 401
+
+    # Test unauthorized
+    api_client.force_authenticate(user=user)
+    response = api_client.delete(f'/api/dandisets/{ds.identifier}/uploads/')
+    assert response.status_code == 403
+
+    assert ds.uploads.count() == 1
+
+
+@pytest.mark.django_db
+def test_dandiset_rest_clear_active_uploads(
+    authenticated_api_client, user, draft_version, upload_factory
+):
+    ds = draft_version.dandiset
+
+    assign_perm('owner', user, ds)
+    upload_factory(dandiset=ds)
+
+    response = authenticated_api_client.get(f'/api/dandisets/{ds.identifier}/uploads/').json()
+    assert response['count'] == 1
+    assert len(response['results']) == 1
+
+    response = authenticated_api_client.delete(f'/api/dandisets/{ds.identifier}/uploads/')
+    assert response.status_code == 204
+
+    assert ds.uploads.count() == 0
+    response = authenticated_api_client.get(f'/api/dandisets/{ds.identifier}/uploads/').json()
+    assert response['count'] == 0
+    assert len(response['results']) == 0

dandiapi/api/views/dandiset.py

@@ -1,11 +1,12 @@
 from __future__ import annotations
 
+import typing
 from typing import TYPE_CHECKING
 
 from allauth.socialaccount.models import SocialAccount
 from django.contrib.auth.models import User
 from django.db import transaction
-from django.db.models import Count, Max, OuterRef, Subquery, Sum
+from django.db.models import Count, Max, OuterRef, QuerySet, Subquery, Sum
 from django.db.models.functions import Coalesce
 from django.db.models.query_utils import Q
 from django.http import Http404
@@ -41,6 +42,8 @@
     DandisetQueryParameterSerializer,
     DandisetSearchQueryParameterSerializer,
     DandisetSearchResultListSerializer,
+    DandisetUploadSerializer,
+    PaginationQuerySerializer,
     UserSerializer,
     VersionMetadataSerializer,
 )
@@ -49,6 +52,8 @@
 if TYPE_CHECKING:
     from rest_framework.request import Request
 
+    from dandiapi.api.models.upload import Upload
+
 
 class DandisetFilterBackend(filters.OrderingFilter):
     ordering_fields = ['id', 'name', 'modified', 'size']
@@ -155,6 +160,16 @@ def get_queryset(self):
                 queryset = queryset.filter(embargo_status='OPEN')
         return queryset
 
+    def require_owner_perm(self, dandiset: Dandiset):
+        # Raise 401 if unauthenticated
+        if not self.request.user.is_authenticated:
+            raise NotAuthenticated
+
+        # Raise 403 if unauthorized
+        self.request.user = typing.cast(User, self.request.user)
+        if not self.request.user.has_perm('owner', dandiset):
+            raise PermissionDenied
+
     def get_object(self):
         # Alternative to path converters, which DRF doesn't support
         # https://docs.djangoproject.com/en/3.0/topics/http/urls/#registering-custom-path-converters
@@ -168,12 +183,8 @@ def get_object(self):
 
         dandiset = super().get_object()
         if dandiset.embargo_status != Dandiset.EmbargoStatus.OPEN:
-            if not self.request.user.is_authenticated:
-                # Clients must be authenticated to access it
-                raise NotAuthenticated
-            if not self.request.user.has_perm('owner', dandiset):
-                # The user does not have ownership permission
-                raise PermissionDenied
+            self.require_owner_perm(dandiset)
+
         return dandiset
 
     @staticmethod
@@ -453,3 +464,41 @@ def users(self, request, dandiset__pk):  # noqa: C901
                 )
 
         return Response(owners, status=status.HTTP_200_OK)
+
+    @swagger_auto_schema(
+        methods=['GET'],
+        manual_parameters=[DANDISET_PK_PARAM],
+        query_serializer=PaginationQuerySerializer,
+        request_body=no_body,
+        operation_summary='List active/incomplete uploads in this dandiset.',
+    )
+    @action(methods=['GET'], detail=True)
+    def uploads(self, request, dandiset__pk):
+        dandiset: Dandiset = self.get_object()
+
+        # Special case where a "safe" method is access restricted, due to the nature of uploads
+        self.require_owner_perm(dandiset)
+
+        uploads: QuerySet[Upload] = dandiset.uploads.all()
+
+        # Paginate and return
+        page = self.paginate_queryset(uploads)
+        if page is not None:
+            serializer = DandisetUploadSerializer(page, many=True)
+            return self.get_paginated_response(serializer.data)
+
+        serializer = DandisetUploadSerializer(uploads, many=True)
+        return Response(serializer.data)
+
+    @swagger_auto_schema(
+        manual_parameters=[DANDISET_PK_PARAM],
+        request_body=no_body,
+        operation_summary='Delete all active/incomplete uploads in this dandiset.',
+    )
+    @uploads.mapping.delete
+    def clear_uploads(self, request, dandiset__pk):
+        dandiset: Dandiset = self.get_object()
+        self.require_owner_perm(dandiset)
+
+        dandiset.uploads.all().delete()
+        return Response(status=status.HTTP_204_NO_CONTENT)

dandiapi/api/views/serializers.py

@@ -8,7 +8,7 @@
 from drf_yasg.utils import swagger_serializer_method
 from rest_framework import serializers
 
-from dandiapi.api.models import Asset, AssetBlob, AssetPath, Dandiset, Version
+from dandiapi.api.models import Asset, AssetBlob, AssetPath, Dandiset, Upload, Version
 from dandiapi.search.models import AssetSearch
 
 if TYPE_CHECKING:
@@ -299,6 +299,17 @@ def get_contact_person(self, obj):
         return extract_contact_person(obj)
 
 
+class DandisetUploadSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = Upload
+        exclude = [
+            'dandiset',
+            'embargoed',
+            'id',
+            'multipart_upload_id',
+        ]
+
+
 class AssetBlobSerializer(serializers.ModelSerializer):
     class Meta:
         model = AssetBlob
@@ -362,6 +373,11 @@ class AssetPathsQueryParameterSerializer(serializers.Serializer):
     path_prefix = serializers.CharField(default='')
 
 
+class PaginationQuerySerializer(serializers.Serializer):
+    page = serializers.IntegerField(default=1)
+    page_size = serializers.IntegerField(default=100)
+
+
 class AssetFileSerializer(AssetSerializer):
     class Meta(AssetSerializer.Meta):
         fields = ['asset_id', 'url']

web/src/rest.ts

@@ -12,6 +12,7 @@ import type {
   AssetPath,
   Zarr,
   DandisetSearchResult,
+  IncompleteUpload,
 } from '@/types';
 import type {
   Dandiset as DandisetMetadata,
@@ -106,6 +107,27 @@ const dandiRest = {
       throw e;
     }
   },
+  async uploads(identifier: string): Promise<IncompleteUpload[]> {
+    const uploads = []
+    let page = 1;
+
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+      const res = await client.get(`dandisets/${identifier}/uploads/`, {params: { page }});
+
+      uploads.push(...res.data.results);
+      if (res.data.next === null) {
+        break;
+      }
+
+      page += 1;
+    }
+
+    return uploads;
+  },
+  async clearUploads(identifier: string) {
+    await client.delete(`dandisets/${identifier}/uploads/`);
+  },
   async assets(
     identifier: string,
     version: string,

web/src/types/index.ts

@@ -102,3 +102,11 @@ export interface AssetPath {
   aggregate_size: number;
   asset: AssetFile | null;
 }
+
+export interface IncompleteUpload {
+  created: string;
+  blob: string;
+  upload_id: string;
+  etag: string;
+  size: number;
+}