Find versions based on filename instead of s3key

actris-cloudnet · Nov 25, 2024 · 024ec21 · 024ec21
1 parent 7256d30
commit 024ec21
Show file tree

Hide file tree

Showing 6 changed files with 99 additions and 65 deletions.
diff --git a/backend/fixtures/5-regular_file.json b/backend/fixtures/5-regular_file.json
@@ -9,7 +9,8 @@
     "updatedAt": "2020-02-20T10:56:19.382Z",
     "startTime": "2018-11-15T10:56:19.382Z",
     "stopTime": "2018-11-15T11:56:19.382Z",
-    "s3key": "20181115_mace-head_mira.nc",
+    "s3key": "38092c00-161d-4ca2-a29d-628cf8e960f6/20181115_mace-head_mira.nc",
+    "filename": "20181115_mace-head_mira.nc",
     "checksum": "298688b011a511f8f0e9353371cf73ee86f60c89b0e02c6931d8c05542c64cdb",
     "size": 12200657,
     "format": "HDF5 (NetCDF4)",
@@ -31,7 +32,8 @@
     "updatedAt": "2020-02-20T10:56:19.382Z",
     "startTime": "2018-11-15T10:56:19.382Z",
     "stopTime": "2018-11-15T11:56:19.382Z",
-    "s3key": "20180609_mace-head_classification.nc",
+    "s3key": "bde7a35f-03aa-4bff-acfb-b4974ea9f217/20180609_mace-head_classification.nc",
+    "filename": "20180609_mace-head_classification.nc",
     "checksum": "b3142dd3b179e8344b30a7c38d0280eab1d122b8683445006e7691fa88ed2c42",
     "size": 130744,
     "format": "HDF5 (NetCDF4)",
@@ -50,7 +52,8 @@
     "updatedAt": "2020-02-20T10:47:33.775Z",
     "startTime": "2018-11-15T10:56:19.382Z",
     "stopTime": "2018-11-15T11:56:19.382Z",
-    "s3key": "20190901_hyytiala_rpg-fmcw-94.nc",
+    "s3key": "d21d6a9b-6804-4465-a026-74ec429fe17d/20190901_hyytiala_rpg-fmcw-94.nc",
+    "filename": "20190901_hyytiala_rpg-fmcw-94.nc",
     "checksum": "9e7c4d902494a254b80d873f13806fd0f01e83564a13a053d3902db98a372ad1",
     "size": 16027310,
     "format": "HDF5 (NetCDF4)",
@@ -70,7 +73,8 @@
     "updatedAt": "2020-02-20T10:39:58.449Z",
     "startTime": "2018-11-15T10:56:19.382Z",
     "stopTime": "2018-11-15T11:56:19.382Z",
-    "s3key": "20190715_bucharest_categorize.nc",
+    "s3key": "22b32746-faf0-4057-9076-ed2e698dcc34/20190715_bucharest_categorize.nc",
+    "filename": "20190715_bucharest_categorize.nc",
     "checksum": "22d86b26f642a0befdcd7088ff38caab00ea3d95933f4ccc430179b4a29b74d3",
     "size": 7127282,
     "format": "HDF5 (NetCDF4)",
@@ -89,7 +93,8 @@
     "updatedAt": "2020-02-20T10:49:58.449Z",
     "startTime": "2018-11-15T10:56:19.382Z",
     "stopTime": "2018-11-15T11:56:19.382Z",
-    "s3key": "20190715_bucharest_categorize.nc",
+    "s3key": "6cb32746-faf0-4057-9076-ed2e698dcf36/20190715_bucharest_categorize.nc",
+    "filename": "20190715_bucharest_categorize.nc",
     "checksum": "43d86b26f642a0befdcd7088ff38caab00ea3d95933f4ccc430179b4a29b74d3",
     "size": 7127282,
     "format": "HDF5 (NetCDF4)",
@@ -108,7 +113,8 @@
     "updatedAt": "2020-02-20T10:59:58.449Z",
     "startTime": "2018-11-15T10:56:19.382Z",
     "stopTime": "2018-11-15T11:56:19.382Z",
-    "s3key": "20190715_bucharest_categorize.nc",
+    "s3key": "8bb32746-faf0-4057-9076-ed2e698dcf36/20190715_bucharest_categorize.nc",
+    "filename": "20190715_bucharest_categorize.nc",
     "checksum": "64d86b26f642a0befdcd7088ff38caab00ea3d95933f4ccc430179b4a29b74d3",
     "size": 7127282,
     "format": "HDF5 (NetCDF4)",
@@ -127,7 +133,8 @@
     "updatedAt": "2020-02-19T10:59:58.449Z",
     "startTime": "2018-11-15T10:56:19.382Z",
     "stopTime": "2018-11-15T11:56:19.382Z",
-    "s3key": "20190715_bucharest_categorize.nc",
+    "s3key": "1bb32746-faf0-4057-9076-ed2e698dcf36/20190715_bucharest_categorize.nc",
+    "filename": "20190715_bucharest_categorize.nc",
     "checksum": "14d86b26f642a0befdcd7088ff38caab00ea3d95933f4ccc430179b4a29b74d3",
     "size": 7127282,
     "format": "HDF5 (NetCDF4)",
@@ -151,7 +158,8 @@
     "updatedAt": "2020-02-19T10:59:58.449Z",
     "startTime": "2018-11-15T10:56:19.382Z",
     "stopTime": "2018-11-15T11:56:19.382Z",
-    "s3key": "20190715_bucharest_categorize.nc",
+    "s3key": "2bb32746-faf0-4057-9076-ed2e698dcf36/20190715_bucharest_categorize.nc",
+    "filename": "20190715_bucharest_categorize.nc",
     "checksum": "24d86b26f642a0befdcd7088ff38caab00ea3d95933f4ccc430179b4a29b74d3",
     "size": 7127282,
     "format": "HDF5 (NetCDF4)",
@@ -172,7 +180,8 @@
     "updatedAt": "2020-02-19T10:59:58.449Z",
     "startTime": "2018-11-15T10:56:19.382Z",
     "stopTime": "2018-11-15T11:56:19.382Z",
-    "s3key": "legacy/20090716_bucharest_classification.nc",
+    "s3key": "3bb32746-faf0-4057-9076-ed2e698dcf36/legacy/20090716_bucharest_classification.nc",
+    "filename": "legacy/20090716_bucharest_classification.nc",
     "checksum": "44d86b26f642a0befdcd7088ff38caab00ea3d95933f4ccc430179b4a29b74d3",
     "size": 7127282,
     "format": "HDF5 (NetCDF4)",
@@ -192,7 +201,8 @@
     "updatedAt": "2020-02-20T10:39:58.449Z",
     "startTime": "2018-11-15T10:56:19.382Z",
     "stopTime": "2018-11-15T11:56:19.382Z",
-    "s3key": "20210126_bucharest_categorize.nc",
+    "s3key": "52b32746-faf0-4057-9076-ed2e698dcc34/20210126_bucharest_categorize.nc",
+    "filename": "20210126_bucharest_categorize.nc",
     "checksum": "52d86b26f642a0befdcd7088ff38caab00ea3d95933f4ccc430179b4a29b74d3",
     "size": 7127282,
     "format": "HDF5 (NetCDF4)",
@@ -212,7 +222,8 @@
     "updatedAt": "2020-02-21T10:49:58.449Z",
     "startTime": "2018-11-15T10:56:19.382Z",
     "stopTime": "2018-11-15T11:56:19.382Z",
-    "s3key": "20210126_bucharest_categorize.nc",
+    "s3key": "62b32746-faf0-4057-9076-ed2e698dcc34/20210126_bucharest_categorize.nc",
+    "filename": "20210126_bucharest_categorize.nc",
     "checksum": "62d86b26f642a0befdcd7088ff38caab00ea3d95933f4ccc430179b4a29b74d3",
     "size": 7127282,
     "format": "HDF5 (NetCDF4)",
@@ -232,7 +243,8 @@
     "updatedAt": "2020-02-21T10:39:58.449Z",
     "startTime": "2018-11-15T10:56:19.382Z",
     "stopTime": "2018-11-15T11:56:19.382Z",
-    "s3key": "20210126_bucharest_categorize.nc",
+    "s3key": "72b32746-faf0-4057-9076-ed2e698dcc34/20210126_bucharest_categorize.nc",
+    "filename": "20210126_bucharest_categorize.nc",
     "checksum": "72d86b26f642a0befdcd7088ff38caab00ea3d95933f4ccc430179b4a29b74d3",
     "size": 7127282,
     "format": "HDF5 (NetCDF4)",
@@ -252,7 +264,8 @@
     "updatedAt": "2020-02-22T10:39:58.449Z",
     "startTime": "2018-11-15T10:56:19.382Z",
     "stopTime": "2018-11-15T11:56:19.382Z",
-    "s3key": "20210126_bucharest_categorize.nc",
+    "s3key": "82b32746-faf0-4057-9076-ed2e698dcc34/20210126_bucharest_categorize.nc",
+    "filename": "20210126_bucharest_categorize.nc",
     "checksum": "82d86b26f642a0befdcd7088ff38caab00ea3d95933f4ccc430179b4a29b74d3",
     "size": 7127282,
     "format": "HDF5 (NetCDF4)",
@@ -272,7 +285,8 @@
     "updatedAt": "2021-02-22T10:39:58.449Z",
     "startTime": "2021-02-20T10:56:19.382Z",
     "stopTime": "2021-02-20T11:56:19.382Z",
-    "s3key": "20210126_bucharest_radar.nc",
+    "s3key": "acf78456-11b1-41a6-b2de-aa7590a75675/20210126_bucharest_radar.nc",
+    "filename": "20210126_bucharest_radar.nc",
     "checksum": "96307ce8dd4fabbc2353cc3c4f32e384bd70280edceb9fb69509cf2db15ad70c",
     "size": 3327282,
     "format": "HDF5 (NetCDF4)",
@@ -292,7 +306,8 @@
     "updatedAt": "2020-02-19T10:59:58.449Z",
     "startTime": "2018-11-15T10:56:19.382Z",
     "stopTime": "2018-11-15T11:56:19.382Z",
-    "s3key": "20190715_bucharest_lidar.nc",
+    "s3key": "b6de8cf4-8825-47b0-aaa9-4fd413bbb0d7/20190715_bucharest_lidar.nc",
+    "filename": "20190715_bucharest_lidar.nc",
     "checksum": "f5e059df0c0dccecc5a5d07d4306e17b9b2b4272ea176688e0ee5c1f651010df",
     "size": 7127282,
     "format": "HDF5 (NetCDF4)",
@@ -314,7 +329,8 @@
     "updatedAt": "2020-02-19T10:59:58.449Z",
     "startTime": "2020-02-19T07:56:19.382Z",
     "stopTime": "2020-02-19T08:56:19.382Z",
-    "s3key": "20190715_bucharest_radar.nc",
+    "s3key": "f036da43-c19c-4832-99f9-6cc88f3255c5/20190715_bucharest_radar.nc",
+    "filename": "20190715_bucharest_radar.nc",
     "checksum": "e09b61366e9c1a4b2676d07dc9bfccff1436a73e46f1a3a8ea51429218d5cb50",
     "size": 7127282,
     "format": "HDF5 (NetCDF4)",
@@ -337,7 +353,8 @@
     "updatedAt": "2020-02-19T10:59:58.449Z",
     "startTime": "2018-11-15T10:56:19.382Z",
     "stopTime": "2018-11-15T11:56:19.382Z",
-    "s3key": "legacy/20090716_newyork_classification.nc",
+    "s3key": "0afca83a-7b6b-4288-82f6-a59685346617/legacy/20090716_newyork_classification.nc",
+    "filename": "legacy/20090716_newyork_classification.nc",
     "checksum": "65d29c507d0ebeec46a6c93deb8ebe3b86dfd678fc73e1a70816cca2fe279025",
     "size": 7127282,
     "format": "HDF5 (NetCDF4)",

diff --git a/backend/src/routes/file.ts b/backend/src/routes/file.ts
@@ -506,7 +506,7 @@ export class FileRoutes {
 
   async fetchValidVersions(queryRunner: QueryRunner, file: File) {
     return await queryRunner.manager.find(RegularFile, {
-      where: { s3key: s3Key(file), tombstoneReason: IsNull() },
+      where: { filename: s3Key(file), tombstoneReason: IsNull() },
       relations: { product: true, site: true },
       order: { createdAt: "DESC" },
     });

diff --git a/backend/tests/data/file.json b/backend/tests/data/file.json
@@ -6,7 +6,8 @@
   "product": "radar",
   "createdAt": "2020-02-20T10:56:19.382Z",
   "updatedAt": "2020-02-20T10:56:19.382Z",
-  "s3key": "20181115_mace-head_mira.nc",
+  "s3key": "38092c00-161d-4ca2-a29d-628cf8e960f6/20181115_mace-head_mira.nc",
+  "filename": "20181115_mace-head_mira.nc",
   "checksum": "298688b011a511f8f0e9353371cf73ee86f60c89b0e02c6931d8c05542c64cdb",
   "size": 12200657,
   "format": "HDF5 (NetCDF4)",

diff --git a/backend/tests/integration/parallel/__snapshots__/file.test.ts.snap b/backend/tests/integration/parallel/__snapshots__/file.test.ts.snap
@@ -4,7 +4,7 @@ exports[`/api/files/:uuid request succeeds on instrument file 1`] = `
 {
   "checksum": "298688b011a511f8f0e9353371cf73ee86f60c89b0e02c6931d8c05542c64cdb",
   "createdAt": "2020-02-20T10:56:19.382Z",
-  "downloadUrl": "http://localhost:3000/api/download/product/38092c00-161d-4ca2-a29d-628cf8e960f6/20181115_mace-head_mira.nc",
+  "downloadUrl": "http://localhost:3000/api/download/product/38092c00-161d-4ca2-a29d-628cf8e960f6/38092c00-161d-4ca2-a29d-628cf8e960f6/20181115_mace-head_mira.nc",
   "dvasId": null,
   "dvasUpdatedAt": null,
   "errorLevel": null,

diff --git a/backend/tests/integration/parallel/search.test.ts b/backend/tests/integration/parallel/search.test.ts
@@ -223,7 +223,7 @@ describe("/api/files", () => {
     };
     const res = await axios.get(url, payload);
     return expect(res.data[0]).toMatchObject({
-      s3path: "/cloudnet-product/legacy/20090716_bucharest_classification.nc",
+      s3path: "/cloudnet-product/3bb32746-faf0-4057-9076-ed2e698dcf36/legacy/20090716_bucharest_classification.nc",
     });
   });
 

diff --git a/backend/tests/integration/sequential/file.test.ts b/backend/tests/integration/sequential/file.test.ts
@@ -731,21 +731,23 @@ describe("DELETE /api/files/", () => {
       volatile: false,
       pid: "https://hdl.handle.net/123/pid1",
     });
-    const file2 = await putDummyFile({
-      product: "radar",
-      volatile: false,
-      pid: "https://hdl.handle.net/123/pid2",
-      version: true,
-    });
+    const file2 = await putDummyFile(
+      {
+        product: "radar",
+        volatile: false,
+        pid: "https://hdl.handle.net/123/pid2",
+      },
+      true,
+    );
     await expect(fileRepo.findOneBy({ uuid: file1.uuid })).resolves.toBeTruthy();
     await expect(fileRepo.findOneBy({ uuid: file2.uuid })).resolves.toBeTruthy();
     expect(await searchFileRepo.existsBy({ uuid: file1.uuid })).toBeFalsy();
     expect(await searchFileRepo.existsBy({ uuid: file2.uuid })).toBeTruthy();
     await expect(deleteFile(file2.uuid, false, false, "Blah")).resolves.toMatchObject({ status: 200 });
     await expect(fileRepo.findOneBy({ uuid: file1.uuid })).resolves.toBeTruthy();
     await expect(fileRepo.findOneBy({ uuid: file2.uuid })).resolves.toBeTruthy();
-    expect(await searchFileRepo.existsBy({ uuid: file1.uuid })).toBeTruthy();
     expect(await searchFileRepo.existsBy({ uuid: file2.uuid })).toBeFalsy();
+    expect(await searchFileRepo.existsBy({ uuid: file1.uuid })).toBeTruthy();
   });
 
   it("test returning to a legacy file in delete", async () => {
@@ -755,12 +757,14 @@ describe("DELETE /api/files/", () => {
       pid: "https://hdl.handle.net/123/pid1",
       legacy: true,
     });
-    const file2 = await putDummyFile({
-      product: "radar",
-      volatile: false,
-      pid: "https://hdl.handle.net/123/pid2",
-      version: true,
-    });
+    const file2 = await putDummyFile(
+      {
+        product: "radar",
+        volatile: false,
+        pid: "https://hdl.handle.net/123/pid2",
+      },
+      true,
+    );
     await expect(fileRepo.findOneBy({ uuid: file1.uuid })).resolves.toBeTruthy();
     await expect(fileRepo.findOneBy({ uuid: file2.uuid })).resolves.toBeTruthy();
     expect(await searchFileRepo.existsBy({ uuid: file1.uuid })).toBeFalsy();
@@ -777,11 +781,13 @@ describe("DELETE /api/files/", () => {
       volatile: false,
       pid: "https://hdl.handle.net/123/pid1",
     });
-    const file2 = await putDummyFile({
-      volatile: false,
-      pid: "https://hdl.handle.net/123/pid2",
-      version: true,
-    });
+    const file2 = await putDummyFile(
+      {
+        volatile: false,
+        pid: "https://hdl.handle.net/123/pid2",
+      },
+      true,
+    );
     await expect(deleteFile(file1.uuid, false, false, "Blah")).resolves.toMatchObject({ status: 200 });
     expect(await searchFileRepo.existsBy({ uuid: file1.uuid })).toBeFalsy();
     expect(await searchFileRepo.existsBy({ uuid: file2.uuid })).toBeTruthy();
@@ -792,16 +798,20 @@ describe("DELETE /api/files/", () => {
       volatile: false,
       pid: "https://hdl.handle.net/123/pid1",
     });
-    const file2 = await putDummyFile({
-      volatile: false,
-      pid: "https://hdl.handle.net/123/pid2",
-      version: true,
-    });
-    const file3 = await putDummyFile({
-      volatile: false,
-      pid: "https://hdl.handle.net/123/pid3",
-      version: true,
-    });
+    const file2 = await putDummyFile(
+      {
+        volatile: false,
+        pid: "https://hdl.handle.net/123/pid2",
+      },
+      true,
+    );
+    const file3 = await putDummyFile(
+      {
+        volatile: false,
+        pid: "https://hdl.handle.net/123/pid3",
+      },
+      true,
+    );
     await expect(deleteFile(file2.uuid, false, false, "Blah")).resolves.toMatchObject({ status: 200 });
     await expect(deleteFile(file3.uuid, false, false, "Blah")).resolves.toMatchObject({ status: 200 });
     expect(await searchFileRepo.existsBy({ uuid: file1.uuid })).toBeTruthy();
@@ -814,24 +824,28 @@ describe("DELETE /api/files/", () => {
       volatile: false,
       pid: "https://hdl.handle.net/123/pid1",
     });
-    const file2 = await putDummyFile({
-      volatile: false,
-      pid: "https://hdl.handle.net/123/pid2",
-      version: true,
-    });
+    const file2 = await putDummyFile(
+      {
+        volatile: false,
+        pid: "https://hdl.handle.net/123/pid2",
+      },
+      true,
+    );
     const catFile1 = await putDummyFile({
       product: "categorize",
       volatile: false,
       sourceFileIds: [file1.uuid],
       pid: "https://hdl.handle.net/123/pid3",
     });
-    const catFile2 = await putDummyFile({
-      product: "categorize",
-      volatile: false,
-      sourceFileIds: [file2.uuid],
-      pid: "https://hdl.handle.net/123/pid4",
-      version: true,
-    });
+    const catFile2 = await putDummyFile(
+      {
+        product: "categorize",
+        volatile: false,
+        sourceFileIds: [file2.uuid],
+        pid: "https://hdl.handle.net/123/pid4",
+      },
+      true,
+    );
 
     expect(await searchFileRepo.existsBy({ uuid: file2.uuid })).toBeTruthy();
     expect(await searchFileRepo.existsBy({ uuid: catFile2.uuid })).toBeTruthy();
@@ -859,18 +873,20 @@ describe("DELETE /api/files/", () => {
       volatile: boolean;
       sourceFileIds: string[];
       pid: string;
-      version: boolean;
       legacy: boolean;
     }> = {},
+    version: boolean = false,
   ) {
-    const statusCode = options.version ? 200 : 201;
+    const statusCode = version ? 200 : 201;
     const { product = "radar", volatile = true, pid = null, legacy = false } = options;
     const fileFix = legacy ? "legacy/" : "";
+    const uuid = uuidGen.v4();
+    const filename = `20181115_mace-head_${product}.nc`;
     const file = {
       ...volatileFile,
       ...options,
-      ...{ uuid: uuidGen.v4(), product, volatile },
-      s3key: `${fileFix}20181115_mace-head_${product}.nc`,
+      ...{ uuid, product, volatile, filename },
+      s3key: `${uuid}/${fileFix}${filename}`,
       checksum: generateHash(),
     };
     if (pid) file.pid = pid;