Release OSIS-Document

README.md

@@ -51,6 +51,8 @@ OSIS_DOCUMENT_UPLOAD_LIMIT = '10/minute'
 OSIS_DOCUMENT_TOKEN_MAX_AGE = 60 * 15
 # A temporary upload max age (in seconds) after which it may be deleted by the celery task
 OSIS_DOCUMENT_TEMP_UPLOAD_MAX_AGE = 60 * 15
+# A deleted upload max age (in seconds) after which it may be deleted by the celery task (default = 15 days)
+OSIS_DOCUMENT_DELETED_UPLOAD_MAX_AGE = 60 * 60 * 24 * 15
 # Upload max age (in seconds) for export expiration policy (default = 15 days)
 OSIS_DOCUMENT_EXPORT_EXPIRATION_POLICY_AGE = 60 * 60 * 24 * 15
 # When used on multiple servers, set the domains on which raw files may be displayed (for Content Security Policy)

frontend/DocumentUploader.vue

@@ -258,6 +258,9 @@ export default defineComponent({
         this.fileList[this.indexGenerated] = file;
         this.tokens[this.indexGenerated] = null;
       });
+
+      // Reset the input value to allow to upload the same file multiple times
+      (this.$refs.fileInput as HTMLInputElement).value = '';
     },
   },
 });

osis_document/api/serializers.py

@@ -241,3 +241,10 @@ class ProgressAsyncPostProcessingResponseSerializer(serializers.Serializer):
         help_text="A dictionary containing the status of the post-processing wanted, the percentage of progress of the post-processing and optionally a boolean if the post-processing failed",
         required=True
     )
+
+
+class DeclareFilesAsDeletedSerializer(serializers.Serializer):
+    files = serializers.ListField(
+        help_text="A list of files UUID",
+        required=True,
+    )

osis_document/api/utils.py

@@ -23,7 +23,7 @@
 #    see http://www.gnu.org/licenses/.
 #
 # ##############################################################################
-from typing import Union, List, Dict
+from typing import Union, List, Dict, Iterable
 from urllib.parse import urlparse
 from uuid import UUID
 
@@ -36,6 +36,8 @@
 from rest_framework.views import APIView
 
 
+
+
 def get_remote_metadata(token: str) -> Union[dict, None]:
     """Given a token, return the remote metadata."""
     import requests
@@ -216,6 +218,25 @@ def launch_post_processing(
     return response.json() if not async_post_processing else response
 
 
+def declare_remote_files_as_deleted(
+    uuid_list: Iterable[UUID]
+):
+    import requests
+
+    url = "{}declare-files-as-deleted".format(settings.OSIS_DOCUMENT_BASE_URL)
+    data = {'files': [str(uuid) for uuid in uuid_list]}
+    response = requests.post(
+        url,
+        json=data,
+        headers={'X-Api-Key': settings.OSIS_DOCUMENT_API_SHARED_SECRET},
+    )
+    if response.status_code != status.HTTP_204_NO_CONTENT:
+        import logging
+
+        logger = logging.getLogger(settings.DEFAULT_LOGGER)
+        logger.error("An error occured when calling declare-files-as-deleted: {}".format(response.text))
+
+
 def get_progress_async_post_processing(uuid: str, wanted_post_process: str = None):
     """Given an uuid and a type of post-processing,
         returns an int corresponding to the post-processing progress percentage

osis_document/api/views/__init__.py

@@ -30,7 +30,7 @@
 from .rotate import RotateImageView
 from .security import DeclareFileAsInfectedView
 from .token import GetTokenView, GetTokenListView
-from .upload import ConfirmUploadView, RequestUploadView
+from .upload import ConfirmUploadView, RequestUploadView, DeclareFilesAsDeletedView
 
 __all__ = [
     "RawFileView",
@@ -43,6 +43,7 @@
     "GetTokenListView",
     "RotateImageView",
     "DeclareFileAsInfectedView",
+    "DeclareFilesAsDeletedView",
     'PostProcessingView',
     "SaveEditorView",
     "GetProgressAsyncPostProcessingView",

osis_document/api/views/metadata.py

@@ -35,7 +35,7 @@
 from osis_document.api import serializers
 from osis_document.api.schema import DetailedAutoSchema
 from osis_document.api.utils import CorsAllowOriginMixin
-from osis_document.enums import DocumentError
+from osis_document.enums import DocumentError, FileStatus
 from osis_document.models import Token
 from osis_document.utils import get_metadata, get_upload_metadata
 
@@ -139,6 +139,8 @@ def post(self, *args, **kwargs):
         tokens = Token.objects.filter(
             token__in=self.request.data,
             expires_at__gt=Now(),
+        ).exclude(
+            upload__status=FileStatus.DELETED.name
         ).select_related('upload')
 
         for token in tokens:

osis_document/api/views/token.py

@@ -59,7 +59,7 @@ class GetTokenView(CorsAllowOriginMixin, generics.CreateAPIView):
 
     name = 'get-token'
     serializer_class = serializers.TokenSerializer
-    queryset = Upload.objects.all()
+    queryset = Upload.objects.all().exclude(status=FileStatus.DELETED.name)
     authentication_classes = []
     permission_classes = [APIKeyPermission]
     token_access = None

osis_document/api/views/upload.py

@@ -23,6 +23,7 @@
 #    see http://www.gnu.org/licenses/.
 #
 # ##############################################################################
+import datetime
 
 from django.conf import settings
 from django.core.exceptions import FieldError
@@ -37,6 +38,7 @@
 from osis_document.api.permissions import APIKeyPermission
 from osis_document.api.schema import DetailedAutoSchema
 from osis_document.api.utils import CorsAllowOriginMixin
+from osis_document.enums import FileStatus
 from osis_document.models import Upload
 from osis_document.utils import calculate_hash, confirm_upload, get_token
 
@@ -136,3 +138,49 @@ def post(self, *args, **kwargs):
         except FieldError as e:
             return Response({'error': str(e)}, status.HTTP_400_BAD_REQUEST)
         return Response({'uuid': uuid}, status.HTTP_201_CREATED)
+
+
+class DeclareFilesAsDeletedSchema(DetailedAutoSchema):  # pragma: no cover
+    serializer_mapping = {
+        'POST': serializers.DeclareFilesAsDeletedSerializer,
+    }
+
+    def get_operation_id(self, path, method):
+        return 'declareFilesAsDeleted'
+
+    def get_responses(self, path, method):
+        responses = super().get_responses(path, method)
+        del responses['201']
+        responses['204'] = {
+            'description': 'No content',
+        }
+        return responses
+
+    def get_operation(self, path, method):
+        operation = super().get_operation(path, method)
+        operation['security'] = [{"ApiKeyAuth": []}]
+        return operation
+
+
+class DeclareFilesAsDeletedView(CorsAllowOriginMixin, APIView):
+    name = 'declare-files-as-deleted'
+    authentication_classes = []
+    permission_classes = [APIKeyPermission]
+    schema = DeclareFilesAsDeletedSchema()
+
+    def post(self, *args, **kwargs):
+        input_serializer_data = serializers.DeclareFilesAsDeletedSerializer(
+            data={
+                **self.request.data,
+            }
+        )
+        input_serializer_data.is_valid(raise_exception=True)
+        validated_data = input_serializer_data.validated_data
+
+        Upload.objects.filter(uuid__in=validated_data['files']).update(
+            status=FileStatus.DELETED.name,
+            expires_at=datetime.date.today() + datetime.timedelta(
+                seconds=settings.OSIS_DOCUMENT_DELETED_UPLOAD_MAX_AGE
+            ),
+        )
+        return Response(status=status.HTTP_204_NO_CONTENT)

osis_document/apps.py

@@ -43,6 +43,10 @@ def ready(self):
             'OSIS_DOCUMENT_EXPORT_EXPIRATION_POLICY_AGE',
             60 * 60 * 24 * 15,
         )
-
+        settings.OSIS_DOCUMENT_DELETED_UPLOAD_MAX_AGE = getattr(
+            settings,
+            'OSIS_DOCUMENT_DELETED_UPLOAD_MAX_AGE',
+            60 * 60 * 24 * 15,
+        )
         # Add FileFieldSerializer the default_serializer_mapping
         ModelSerializer.serializer_field_mapping[FileField] = FileFieldSerializer

osis_document/contrib/fields.py

@@ -23,13 +23,17 @@
 #  see http://www.gnu.org/licenses/.
 #
 # ##############################################################################
-
+import uuid
 from os.path import dirname
+from typing import Set, List, Union
 
 from django.contrib.postgres.fields import ArrayField
 from django.contrib.postgres.validators import ArrayMinLengthValidator
+from django.core.exceptions import ObjectDoesNotExist
 from django.db import models
 from django.utils.translation import gettext_lazy as _
+from uuid import UUID
+
 from osis_document.contrib.forms import FileUploadField
 from osis_document.utils import generate_filename
 from osis_document.enums import DocumentExpirationPolicy
@@ -97,27 +101,49 @@ def formfield(self, **kwargs):
         )
 
     def pre_save(self, model_instance, add):
-        """Convert all writing tokens to UUIDs by remotely confirming their upload, leaving existing uuids"""
-        value = [
-            self._confirm_upload(model_instance, token) if isinstance(token, str) else token
-            for token in (getattr(model_instance, self.attname) or [])
-        ]
+        """
+        Convert all writing tokens to UUIDs by remotely confirming their upload, leaving existing uuids
+        and deleting old documents
+        """
+        try:
+            previous_values = self.model.objects.values_list(self.attname).get(pk=model_instance.pk)[0] or []
+        except ObjectDoesNotExist:
+            previous_values = []
+
+        attvalues = getattr(model_instance, self.attname) or []
+        files_confirmed = self._confirm_multiple_upload(model_instance, attvalues, previous_values)
         if self.post_processing:
-            self._post_processing(uuid_list=value)
-        setattr(model_instance, self.attname, value)
-        return value
+            self._post_processing(uuid_list=files_confirmed)
+        setattr(model_instance, self.attname, files_confirmed)
+        return files_confirmed
 
-    def _confirm_upload(self, model_instance, token):
-        """Call the remote API to confirm the upload of a token"""
-        from osis_document.api.utils import confirm_remote_upload, get_remote_metadata
+    def _confirm_multiple_upload(
+        self,
+        model_instance,
+        attvalues: List[Union[str, UUID]],
+        previous_values: List[UUID],
+    ):
+        """Call the remote API to confirm multiple upload and delete old file if replaced"""
+        from osis_document.api.utils import confirm_remote_upload, get_several_remote_metadata, \
+            declare_remote_files_as_deleted
 
-        # Get the current filename by interrogating API
-        filename = get_remote_metadata(token)['name']
-        return confirm_remote_upload(
-            token=token,
-            upload_to=dirname(generate_filename(model_instance, filename, self.upload_to)),
-            document_expiration_policy=self.document_expiration_policy,
-        )
+        files_to_keep = [token for token in attvalues if not isinstance(token, str)]  # UUID
+
+        tokens = [token for token in attvalues if isinstance(token, str)]
+        metadata_by_token = get_several_remote_metadata(tokens) if tokens else {}
+        for token in tokens:
+            filename = metadata_by_token[token]['name']
+            file_uuid = confirm_remote_upload(
+                token=token,
+                upload_to=dirname(generate_filename(model_instance, filename, self.upload_to)),
+                document_expiration_policy=self.document_expiration_policy,
+            )
+            files_to_keep.append(UUID(file_uuid))
+
+        files_to_declare_as_deleted = set(previous_values) - set(files_to_keep)
+        if files_to_declare_as_deleted:
+            declare_remote_files_as_deleted(files_to_declare_as_deleted)
+        return files_to_keep
 
     def _post_processing(self, uuid_list: list):
         from osis_document.api.utils import launch_post_processing

osis_document/contrib/post_processing/merger.py

@@ -36,7 +36,7 @@
 from osis_document.enums import PostProcessingType
 from osis_document.exceptions import FormatInvalidException, MissingFileException, InvalidMergeFileDimension
 from osis_document.models import Upload
-from osis_document.utils import stringify_uuid_and_check_uuid_validity
+from osis_document.utils import stringify_uuid_and_check_uuid_validity, FILENAME_MAX_LENGTH
 
 
 class Merger(Processor):
@@ -108,8 +108,10 @@ def _merge_and_change_pages_dimension(writer_instance: PdfWriter, pages: List[Pa
     @staticmethod
     def _get_output_filename(output_filename: str = None):
         if output_filename:
-            return f"{output_filename}{uuid.uuid4()}.pdf"
-        return f"merge_{uuid.uuid4()}.pdf"
+            filename = f"{output_filename}{uuid.uuid4()}"
+        else:
+            filename = f"merge_{uuid.uuid4()}"
+        return f"{filename[:FILENAME_MAX_LENGTH - 4]}.pdf"
 
 
 merger = Merger()

osis_document/enums.py

@@ -60,6 +60,7 @@ def __deepcopy__(self, memodict: Dict = None) -> 'ChoiceEnum':
 class FileStatus(ChoiceEnum):
     REQUESTED = _('Requested')
     UPLOADED = _('Uploaded')
+    DELETED = _('Deleted')
     INFECTED = _('Infected')