S3 upload workflow

app/settings.py

@@ -181,6 +181,7 @@
 # ------------------------------------------------------------------------------
 
 CORS_ORIGIN_WHITELIST = [
+    "http://127.0.0.1:8000",
     "http://localhost:8000",
     "http://localhost:8080",
     "https://quiz-anthropocene.netlify.com",
@@ -270,6 +271,23 @@
 QUESTION_FOLDER_NAME = "questions"
 QUIZ_FOLDER_NAME = "quizs"
 
+STORAGE_UPLOAD_KINDS = {
+    "default": {
+        "allowed_mime_types": ["image/png", "image/svg+xml", "image/gif", "image/jpg", "image/jpeg"],  # ["image/*"] ?
+        "upload_expiration": 60 * 60,  # in seconds
+        "key_path": "default",  # appended before the file key. No backslash!
+        "max_files": 1,  # 3,
+        "max_file_size": 5,  # in mb
+        "timeout": 20000,  # in ms
+    },
+    "question_answer_image": {
+        "key_path": QUESTION_FOLDER_NAME,
+    },
+    "quiz_image_background": {
+        "key_path": QUIZ_FOLDER_NAME,
+    },
+}
+
 
 # Django Bootstrap5
 # https://django-bootstrap5.readthedocs.io/

core/static/js/s3_upload.js

@@ -0,0 +1,171 @@
+"use strict";
+
+const extensionToContentTypeMapping = {
+  'png': 'image/png',
+  'PNG': 'image/png',
+  'svg': 'image/svg+xml',
+  'gif': 'image/gif',
+  'jpg': 'image/jpg',
+  'JPG': 'image/jpg',
+  'jpeg': 'image/jpeg',
+}
+
+// Arguments
+// - callbackLocationSelector: input field where the location URL will be provided after success (ex. "#foo").
+// - dropzoneSelector: transform this element into a drag and drop zone.
+// - s3FormValuesId: ID of DOM element that contains the JSON values of the form (eg. URL, fields, etc)
+// - s3UploadConfigId: ID of DOM element that contains the JSON values of the S3 config (eg. max file size, timeout, etc).
+
+window.s3UploadInit = function s3UploadInit({
+  dropzoneSelector = "#answer_image_form",
+  callbackLocationSelector = "",
+  s3FormValuesId = "s3-form-values",
+  s3UploadConfigId = "s3-upload-config",
+  sentryInternalUrl = "",
+  sentryCsrfToken = "",
+} = {}) {
+  const formValues = JSON.parse(
+    document.getElementById(s3FormValuesId).textContent
+  );
+  const uploadConfig = JSON.parse(
+    document.getElementById(s3UploadConfigId).textContent
+  );
+
+  // When a file is added to the drop zone, send a POST request to this URL.
+  const formUrl = formValues["url"];
+
+  // Submit button to be disabled during file processing
+  const submitButton = $("button[type='submit']");
+
+  // S3 form params sent when a new file is added to the drop zone.
+  let formParams = formValues["fields"];
+
+  // Appended before the file name. The final slash is added later.
+  const keyPath = uploadConfig["key_path"];
+
+  // Dropzone configuration
+  const dropzoneConfig = {
+    url: formUrl,
+    params: formParams,
+    maxFilesize: uploadConfig["max_file_size"], // in MB
+    timeout: uploadConfig["timeout"], // default 3000, in ms
+    maxFiles: uploadConfig["max_files"],
+    acceptedFiles: uploadConfig["allowed_mime_types"],
+    // UI config
+    addRemoveLinks: true,
+    // translations
+    dictFallbackMessage: "Ce navigateur n'est pas compatible",
+    dictFileTooBig: "Fichier trop volumineux",
+    dictInvalidFileType: "Type de fichier non pris en charge",
+    dictResponseError: "Erreur technique. Merci de recommencer.",
+    dictCancelUpload: "Annuler",
+    dictUploadCanceled: "Annulé",
+    dictCancelUploadConfirmation: "Voulez-vous vraiment annuler le transfert ?",
+    dictRemoveFile: "Supprimer",
+    dictRemoveFileConfirmation: "Voulez-vous vraiment supprimer le fichier ?",
+    dictMaxFilesExceeded: "Un seul fichier autorisé à la fois",
+    // the function will be used to rename the file.name before appending it to the formData
+    renameFile: function (file) {
+      const extension = file.name.split(".").pop();
+      const filename = Dropzone.uuidv4();
+      const fileKey = `${keyPath}/${filename}.${extension}`;
+      // Add a file key to options params so that it's sent as an input field on POST.
+      this.params["key"] = fileKey;
+      // We also send the Content-Type depending on the image type.
+      // Important because by default the Content-Type will be 'application/octet-stream' (when fetched, images would be downloaded automatically instead of being displayed)
+      this.params["Content-Type"] = extensionToContentTypeMapping[extension];
+      return fileKey;
+    },
+  };
+
+  // By default, Dropzone attaches to any component having the "dropzone" class.
+  // Turn off this behavior to control all the aspects "manually".
+  Dropzone.autoDiscover = false;
+
+  const dropzone = new Dropzone(dropzoneSelector, dropzoneConfig);
+
+  // Display a help message when the user tries to
+  // submit the form during file transfer.
+  // submitButton.tooltip({ title: "Veuillez attendre la fin du transfert" });
+  // Enable it later, during file transfer.
+  // submitButton.tooltip("disable");
+
+  // Events
+  dropzone.on("addedfile", function (file) {
+    // submitButton.tooltip("enable");
+    submitButton.prop("disabled", true);
+    submitButton.addClass("btn-secondary");
+  });
+
+  // Called when the upload was either successful or erroneous.
+  dropzone.on("complete", function (file) {
+    console.log("complete")
+    // submitButton.tooltip("disable");
+    submitButton.prop("disabled", false);
+    submitButton.removeClass("btn-secondary");
+  });
+
+  dropzone.on("removedfile", function (file) {
+    $(callbackLocationSelector).val("");
+  });
+
+  dropzone.on("success", function (file, xhr, formData) {
+    const location = `${formUrl}/${file.upload.filename}`;
+    // Prevent a selector mistake from being silent.
+    if ($(callbackLocationSelector).length === 0) {
+      this._handleUploadError(
+        [file],
+        xhr,
+        "Ce document n'a pas pu être envoyé à cause d'un problème technique. Nous vous invitons à contacter notre support."
+      );
+    }
+    $(callbackLocationSelector).val(location);
+  });
+
+  dropzone.on("error", function (file, errorMessage, xhr) {
+    let statusCode = 500;
+
+    if (typeof errorMessage === "string") {
+      if (errorMessage.includes("timedout")) {
+        // Override default English message and don't send the error to Sentry.
+        file.previewElement.querySelectorAll('[data-dz-errormessage]')[0].textContent = "Erreur technique. Merci de recommencer.";
+        return
+      }
+    }
+    else {
+      // errorMessage is a JSON object. Display a nice message to the user instead of [object Object].
+      file.previewElement.querySelectorAll('[data-dz-errormessage]')[0].textContent = "Erreur technique. Merci de recommencer.";
+    }
+
+    if (xhr) {
+      statusCode = xhr.status;
+
+      if (statusCode === 0) {
+        // Don't send undefined errors to Sentry.
+        // Might be due to a firewall or to an unreachable network.
+        // See https://stackoverflow.com/questions/872206/what-does-it-mean-when-an-http-request-returns-status-code-0
+        return
+      }
+
+      if (xhr.responseText) {
+        let responseJson = JSON.parse(xhr.responseText);
+        errorMessage = responseJson["Message"];
+        // User waited too long before sending the file.
+        // See base.py > STORAGE_UPLOAD_KINDS > upload expiration
+        if (errorMessage === "Policy expired") {
+          return
+        }
+      }
+
+      // An error occurred with the request. Send it to Sentry to avoid silent bugs.
+      const sentryErrorMessage =
+        `Unable to upload "${file.upload.filename}" ` +
+        `(${file.upload.progress} of ${file.upload.total}) to S3 ${formUrl}: ${errorMessage}`;
+      $.post(sentryInternalUrl, {
+        status_code: statusCode,
+        error_message: sentryErrorMessage,
+        csrfmiddlewaretoken: sentryCsrfToken,
+      });
+    }
+  });
+};

core/utils/s3.py

@@ -74,6 +74,51 @@
 bucket = resource.Bucket(settings.S3_BUCKET_NAME)
 
 
+class S3Upload:
+    def __init__(self, kind="default"):
+        self.config = self.get_config(kind)
+
+    @property
+    def form_values(self):
+        """
+        Returns a dict like this:
+        {
+            "url": "",
+            "fields": {
+                'key': 'key_path',
+                'x-amz-algorithm': 'AWS4-HMAC-SHA256',
+                'x-amz-credential': '',
+                'x-amz-date': '',
+                'policy': '',
+                'x-amz-signature': '',
+            }
+        }
+        """
+        key_path = self.config["key_path"] + "/${filename}"
+        expiration = self.config["upload_expiration"]
+        values_dict = client.generate_presigned_post(
+            settings.S3_BUCKET_NAME,
+            key_path,
+            ExpiresIn=expiration,
+            Conditions=[["starts-with", "$Content-Type", "image/"]],
+        )
+        values_dict["fields"].pop("key")
+        return values_dict
+
+    @staticmethod
+    def get_config(kind):
+        default_options = settings.STORAGE_UPLOAD_KINDS["default"]
+        config = default_options | settings.STORAGE_UPLOAD_KINDS[kind]
+
+        key_path = config["key_path"]
+        if key_path.startswith("/") or key_path.endswith("/"):
+            raise ValueError("key_path should not begin or end with a slash")
+
+        config["allowed_mime_types"] = ",".join(config["allowed_mime_types"])
+
+        return config
+
+
 def get_bucket(bucket_name=settings.S3_BUCKET_NAME):
     bucket = resource.Bucket(bucket_name)
     return bucket

questions/forms.py

@@ -21,6 +21,7 @@ class Meta:
             "hint": forms.Textarea(attrs={"rows": 1}),
             "answer_explanation": forms.Textarea(attrs={"rows": 3}),
             "answer_reading_recommendation": forms.Textarea(attrs={"rows": 1}),
+            "answer_image_url": forms.HiddenInput(),
             "answer_image_explanation": forms.Textarea(attrs={"rows": 1}),
             "answer_extra_info": forms.Textarea(attrs={"rows": 3}),
         }