Merge pull request #721 from opengisch/QF-2910-skip-metadata

Skip downloading the S3 metadata if `skip_metadata` is passed to the API `/files/` list endpoint

docker-app/qfieldcloud/core/tests/test_qgis_file.py

@@ -223,6 +223,73 @@ def test_upload_and_list_file(self):
             "fcc85fb502bd772aa675a0263b5fa665bccd5d8d93349d1dbc9f0f6394dd37b9",
         )
 
+    def test_upload_and_list_file_checksum(self):
+        self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
+
+        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 0)
+        self.assertEqual(
+            Project.objects.get(pk=self.project1.pk).project_filename, None
+        )
+
+        file_path = testdata_path("file.txt")
+        # Push a file
+        response = self.client.post(
+            f"/api/v1/files/{self.project1.id}/file.txt/",
+            {"file": open(file_path, "rb")},
+            format="multipart",
+        )
+        self.assertTrue(status.is_success(response.status_code))
+        self.assertEqual(Project.objects.get(pk=self.project1.pk).files_count, 1)
+
+        # List files without `skip_metadata` param
+        response = self.client.get(f"/api/v1/files/{self.project1.id}/")
+        self.assertTrue(status.is_success(response.status_code))
+
+        json = response.json()
+
+        self.assertEqual(json[0]["name"], "file.txt")
+        self.assertEqual(json[0]["size"], 13)
+        self.assertIn("sha256", json[0])
+        self.assertIn("md5sum", json[0])
+        self.assertEqual(
+            json[0]["sha256"],
+            "8663bab6d124806b9727f89bb4ab9db4cbcc3862f6bbf22024dfa7212aa4ab7d",
+        )
+        self.assertEqual(
+            json[0]["md5sum"],
+            "9af2f8218b150c351ad802c6f3d66abe",
+        )
+
+        # List files with `skip_metadata=0` param
+        response = self.client.get(f"/api/v1/files/{self.project1.id}/?skip_metadata=0")
+        self.assertEqual(json[0]["name"], "file.txt")
+        self.assertEqual(json[0]["size"], 13)
+        self.assertIn("sha256", json[0])
+        self.assertIn("md5sum", json[0])
+        self.assertEqual(
+            json[0]["sha256"],
+            "8663bab6d124806b9727f89bb4ab9db4cbcc3862f6bbf22024dfa7212aa4ab7d",
+        )
+        self.assertEqual(
+            json[0]["md5sum"],
+            "9af2f8218b150c351ad802c6f3d66abe",
+        )
+
+        # List files with `skip_metadata=1` param
+        response = self.client.get(f"/api/v1/files/{self.project1.id}/?skip_metadata=1")
+        self.assertTrue(status.is_success(response.status_code))
+
+        json = response.json()
+
+        self.assertEqual(json[0]["name"], "file.txt")
+        self.assertEqual(json[0]["size"], 13)
+        self.assertNotIn("sha256", json[0])
+        self.assertIn("md5sum", json[0])
+        self.assertEqual(
+            json[0]["md5sum"],
+            "9af2f8218b150c351ad802c6f3d66abe",
+        )
+
     def test_upload_and_list_file_with_space_in_name(self):
         self.client.credentials(HTTP_AUTHORIZATION="Token " + self.token1.key)
 

docker-app/qfieldcloud/core/views/files_views.py

@@ -58,40 +58,50 @@ def get(self, request: Request, projectid: str) -> Response:
             if version.key not in files:
                 files[version.key] = {"versions": []}
 
-            head = version.head()
             path = PurePath(version.key)
             filename = str(path.relative_to(*path.parts[:3]))
             last_modified = version.last_modified.strftime("%d.%m.%Y %H:%M:%S %Z")
 
-            # We cannot be sure of the metadata's first letter case
-            # https://github.com/boto/boto3/issues/1709
-            metadata = head["Metadata"]
-            if "sha256sum" in metadata:
-                sha256sum = metadata["sha256sum"]
+            version_data = {
+                "size": version.size,
+                "md5sum": version.e_tag.replace('"', ""),
+                "version_id": version.version_id,
+                "last_modified": last_modified,
+                "is_latest": version.is_latest,
+                "display": S3ObjectVersion(version.key, version).display,
+            }
+
+            # NOTE Some clients (e.g. QField, QFieldSync) are still requiring the `sha256` key to check whether the files needs to be reuploaded.
+            # Since we do not have control on these old client versions, we need to keep the API backward compatible for some time and assume `skip_metadata=0` by default.
+            skip_metadata_param = request.GET.get("skip_metadata", "0")
+            if skip_metadata_param == "0":
+                skip_metadata = False
             else:
-                sha256sum = metadata["Sha256sum"]
+                skip_metadata = bool(skip_metadata_param)
+
+            if not skip_metadata:
+                head = version.head()
+                # We cannot be sure of the metadata's first letter case
+                # https://github.com/boto/boto3/issues/1709
+                metadata = head["Metadata"]
+                if "sha256sum" in metadata:
+                    sha256sum = metadata["sha256sum"]
+                else:
+                    sha256sum = metadata["Sha256sum"]
+                files[version.key]["sha256"] = sha256sum
+
+                version_data["sha256"] = sha256sum
 
             if version.is_latest:
                 is_attachment = get_attachment_dir_prefix(project, filename) != ""
 
                 files[version.key]["name"] = filename
                 files[version.key]["size"] = version.size
-                files[version.key]["sha256"] = sha256sum
                 files[version.key]["md5sum"] = version.e_tag.replace('"', "")
                 files[version.key]["last_modified"] = last_modified
                 files[version.key]["is_attachment"] = is_attachment
 
-            files[version.key]["versions"].append(
-                {
-                    "size": version.size,
-                    "sha256": sha256sum,
-                    "md5sum": version.e_tag.replace('"', ""),
-                    "version_id": version.version_id,
-                    "last_modified": last_modified,
-                    "is_latest": version.is_latest,
-                    "display": S3ObjectVersion(version.key, version).display,
-                }
-            )
+            files[version.key]["versions"].append(version_data)
 
         result_list = [files[key] for key in files]
         return Response(result_list)