diff --git a/backend/fixtures/5-regular_file.json b/backend/fixtures/5-regular_file.json index 32a827907..8b6bc6fc9 100644 --- a/backend/fixtures/5-regular_file.json +++ b/backend/fixtures/5-regular_file.json @@ -9,7 +9,8 @@ "updatedAt": "2020-02-20T10:56:19.382Z", "startTime": "2018-11-15T10:56:19.382Z", "stopTime": "2018-11-15T11:56:19.382Z", - "s3key": "20181115_mace-head_mira.nc", + "s3key": "38092c00-161d-4ca2-a29d-628cf8e960f6/20181115_mace-head_mira.nc", + "filename": "20181115_mace-head_mira.nc", "checksum": "298688b011a511f8f0e9353371cf73ee86f60c89b0e02c6931d8c05542c64cdb", "size": 12200657, "format": "HDF5 (NetCDF4)", @@ -31,7 +32,8 @@ "updatedAt": "2020-02-20T10:56:19.382Z", "startTime": "2018-11-15T10:56:19.382Z", "stopTime": "2018-11-15T11:56:19.382Z", - "s3key": "20180609_mace-head_classification.nc", + "s3key": "bde7a35f-03aa-4bff-acfb-b4974ea9f217/20180609_mace-head_classification.nc", + "filename": "20180609_mace-head_classification.nc", "checksum": "b3142dd3b179e8344b30a7c38d0280eab1d122b8683445006e7691fa88ed2c42", "size": 130744, "format": "HDF5 (NetCDF4)", @@ -50,7 +52,8 @@ "updatedAt": "2020-02-20T10:47:33.775Z", "startTime": "2018-11-15T10:56:19.382Z", "stopTime": "2018-11-15T11:56:19.382Z", - "s3key": "20190901_hyytiala_rpg-fmcw-94.nc", + "s3key": "d21d6a9b-6804-4465-a026-74ec429fe17d/20190901_hyytiala_rpg-fmcw-94.nc", + "filename": "20190901_hyytiala_rpg-fmcw-94.nc", "checksum": "9e7c4d902494a254b80d873f13806fd0f01e83564a13a053d3902db98a372ad1", "size": 16027310, "format": "HDF5 (NetCDF4)", @@ -70,7 +73,8 @@ "updatedAt": "2020-02-20T10:39:58.449Z", "startTime": "2018-11-15T10:56:19.382Z", "stopTime": "2018-11-15T11:56:19.382Z", - "s3key": "20190715_bucharest_categorize.nc", + "s3key": "22b32746-faf0-4057-9076-ed2e698dcc34/20190715_bucharest_categorize.nc", + "filename": "20190715_bucharest_categorize.nc", "checksum": "22d86b26f642a0befdcd7088ff38caab00ea3d95933f4ccc430179b4a29b74d3", "size": 7127282, "format": "HDF5 (NetCDF4)", @@ -89,7 +93,8 @@ "updatedAt": "2020-02-20T10:49:58.449Z", "startTime": "2018-11-15T10:56:19.382Z", "stopTime": "2018-11-15T11:56:19.382Z", - "s3key": "20190715_bucharest_categorize.nc", + "s3key": "6cb32746-faf0-4057-9076-ed2e698dcf36/20190715_bucharest_categorize.nc", + "filename": "20190715_bucharest_categorize.nc", "checksum": "43d86b26f642a0befdcd7088ff38caab00ea3d95933f4ccc430179b4a29b74d3", "size": 7127282, "format": "HDF5 (NetCDF4)", @@ -108,7 +113,8 @@ "updatedAt": "2020-02-20T10:59:58.449Z", "startTime": "2018-11-15T10:56:19.382Z", "stopTime": "2018-11-15T11:56:19.382Z", - "s3key": "20190715_bucharest_categorize.nc", + "s3key": "8bb32746-faf0-4057-9076-ed2e698dcf36/20190715_bucharest_categorize.nc", + "filename": "20190715_bucharest_categorize.nc", "checksum": "64d86b26f642a0befdcd7088ff38caab00ea3d95933f4ccc430179b4a29b74d3", "size": 7127282, "format": "HDF5 (NetCDF4)", @@ -127,7 +133,8 @@ "updatedAt": "2020-02-19T10:59:58.449Z", "startTime": "2018-11-15T10:56:19.382Z", "stopTime": "2018-11-15T11:56:19.382Z", - "s3key": "20190715_bucharest_categorize.nc", + "s3key": "1bb32746-faf0-4057-9076-ed2e698dcf36/20190715_bucharest_categorize.nc", + "filename": "20190715_bucharest_categorize.nc", "checksum": "14d86b26f642a0befdcd7088ff38caab00ea3d95933f4ccc430179b4a29b74d3", "size": 7127282, "format": "HDF5 (NetCDF4)", @@ -151,7 +158,8 @@ "updatedAt": "2020-02-19T10:59:58.449Z", "startTime": "2018-11-15T10:56:19.382Z", "stopTime": "2018-11-15T11:56:19.382Z", - "s3key": "20190715_bucharest_categorize.nc", + "s3key": "2bb32746-faf0-4057-9076-ed2e698dcf36/20190715_bucharest_categorize.nc", + "filename": "20190715_bucharest_categorize.nc", "checksum": "24d86b26f642a0befdcd7088ff38caab00ea3d95933f4ccc430179b4a29b74d3", "size": 7127282, "format": "HDF5 (NetCDF4)", @@ -172,7 +180,8 @@ "updatedAt": "2020-02-19T10:59:58.449Z", "startTime": "2018-11-15T10:56:19.382Z", "stopTime": "2018-11-15T11:56:19.382Z", - "s3key": "legacy/20090716_bucharest_classification.nc", + "s3key": "3bb32746-faf0-4057-9076-ed2e698dcf36/legacy/20090716_bucharest_classification.nc", + "filename": "legacy/20090716_bucharest_classification.nc", "checksum": "44d86b26f642a0befdcd7088ff38caab00ea3d95933f4ccc430179b4a29b74d3", "size": 7127282, "format": "HDF5 (NetCDF4)", @@ -192,7 +201,8 @@ "updatedAt": "2020-02-20T10:39:58.449Z", "startTime": "2018-11-15T10:56:19.382Z", "stopTime": "2018-11-15T11:56:19.382Z", - "s3key": "20210126_bucharest_categorize.nc", + "s3key": "52b32746-faf0-4057-9076-ed2e698dcc34/20210126_bucharest_categorize.nc", + "filename": "20210126_bucharest_categorize.nc", "checksum": "52d86b26f642a0befdcd7088ff38caab00ea3d95933f4ccc430179b4a29b74d3", "size": 7127282, "format": "HDF5 (NetCDF4)", @@ -212,7 +222,8 @@ "updatedAt": "2020-02-21T10:49:58.449Z", "startTime": "2018-11-15T10:56:19.382Z", "stopTime": "2018-11-15T11:56:19.382Z", - "s3key": "20210126_bucharest_categorize.nc", + "s3key": "62b32746-faf0-4057-9076-ed2e698dcc34/20210126_bucharest_categorize.nc", + "filename": "20210126_bucharest_categorize.nc", "checksum": "62d86b26f642a0befdcd7088ff38caab00ea3d95933f4ccc430179b4a29b74d3", "size": 7127282, "format": "HDF5 (NetCDF4)", @@ -232,7 +243,8 @@ "updatedAt": "2020-02-21T10:39:58.449Z", "startTime": "2018-11-15T10:56:19.382Z", "stopTime": "2018-11-15T11:56:19.382Z", - "s3key": "20210126_bucharest_categorize.nc", + "s3key": "72b32746-faf0-4057-9076-ed2e698dcc34/20210126_bucharest_categorize.nc", + "filename": "20210126_bucharest_categorize.nc", "checksum": "72d86b26f642a0befdcd7088ff38caab00ea3d95933f4ccc430179b4a29b74d3", "size": 7127282, "format": "HDF5 (NetCDF4)", @@ -252,7 +264,8 @@ "updatedAt": "2020-02-22T10:39:58.449Z", "startTime": "2018-11-15T10:56:19.382Z", "stopTime": "2018-11-15T11:56:19.382Z", - "s3key": "20210126_bucharest_categorize.nc", + "s3key": "82b32746-faf0-4057-9076-ed2e698dcc34/20210126_bucharest_categorize.nc", + "filename": "20210126_bucharest_categorize.nc", "checksum": "82d86b26f642a0befdcd7088ff38caab00ea3d95933f4ccc430179b4a29b74d3", "size": 7127282, "format": "HDF5 (NetCDF4)", @@ -272,7 +285,8 @@ "updatedAt": "2021-02-22T10:39:58.449Z", "startTime": "2021-02-20T10:56:19.382Z", "stopTime": "2021-02-20T11:56:19.382Z", - "s3key": "20210126_bucharest_radar.nc", + "s3key": "acf78456-11b1-41a6-b2de-aa7590a75675/20210126_bucharest_radar.nc", + "filename": "20210126_bucharest_radar.nc", "checksum": "96307ce8dd4fabbc2353cc3c4f32e384bd70280edceb9fb69509cf2db15ad70c", "size": 3327282, "format": "HDF5 (NetCDF4)", @@ -292,7 +306,8 @@ "updatedAt": "2020-02-19T10:59:58.449Z", "startTime": "2018-11-15T10:56:19.382Z", "stopTime": "2018-11-15T11:56:19.382Z", - "s3key": "20190715_bucharest_lidar.nc", + "s3key": "b6de8cf4-8825-47b0-aaa9-4fd413bbb0d7/20190715_bucharest_lidar.nc", + "filename": "20190715_bucharest_lidar.nc", "checksum": "f5e059df0c0dccecc5a5d07d4306e17b9b2b4272ea176688e0ee5c1f651010df", "size": 7127282, "format": "HDF5 (NetCDF4)", @@ -314,7 +329,8 @@ "updatedAt": "2020-02-19T10:59:58.449Z", "startTime": "2020-02-19T07:56:19.382Z", "stopTime": "2020-02-19T08:56:19.382Z", - "s3key": "20190715_bucharest_radar.nc", + "s3key": "f036da43-c19c-4832-99f9-6cc88f3255c5/20190715_bucharest_radar.nc", + "filename": "20190715_bucharest_radar.nc", "checksum": "e09b61366e9c1a4b2676d07dc9bfccff1436a73e46f1a3a8ea51429218d5cb50", "size": 7127282, "format": "HDF5 (NetCDF4)", @@ -337,7 +353,8 @@ "updatedAt": "2020-02-19T10:59:58.449Z", "startTime": "2018-11-15T10:56:19.382Z", "stopTime": "2018-11-15T11:56:19.382Z", - "s3key": "legacy/20090716_newyork_classification.nc", + "s3key": "0afca83a-7b6b-4288-82f6-a59685346617/legacy/20090716_newyork_classification.nc", + "filename": "legacy/20090716_newyork_classification.nc", "checksum": "65d29c507d0ebeec46a6c93deb8ebe3b86dfd678fc73e1a70816cca2fe279025", "size": 7127282, "format": "HDF5 (NetCDF4)", diff --git a/backend/src/routes/file.ts b/backend/src/routes/file.ts index 868785c8d..5a743a8a0 100644 --- a/backend/src/routes/file.ts +++ b/backend/src/routes/file.ts @@ -506,7 +506,7 @@ export class FileRoutes { async fetchValidVersions(queryRunner: QueryRunner, file: File) { return await queryRunner.manager.find(RegularFile, { - where: { s3key: s3Key(file), tombstoneReason: IsNull() }, + where: { filename: s3Key(file), tombstoneReason: IsNull() }, relations: { product: true, site: true }, order: { createdAt: "DESC" }, }); diff --git a/backend/tests/data/file.json b/backend/tests/data/file.json index 1d8e5e973..ae47949b4 100644 --- a/backend/tests/data/file.json +++ b/backend/tests/data/file.json @@ -6,7 +6,8 @@ "product": "radar", "createdAt": "2020-02-20T10:56:19.382Z", "updatedAt": "2020-02-20T10:56:19.382Z", - "s3key": "20181115_mace-head_mira.nc", + "s3key": "38092c00-161d-4ca2-a29d-628cf8e960f6/20181115_mace-head_mira.nc", + "filename": "20181115_mace-head_mira.nc", "checksum": "298688b011a511f8f0e9353371cf73ee86f60c89b0e02c6931d8c05542c64cdb", "size": 12200657, "format": "HDF5 (NetCDF4)", diff --git a/backend/tests/integration/parallel/__snapshots__/file.test.ts.snap b/backend/tests/integration/parallel/__snapshots__/file.test.ts.snap index fdfe30369..93030db9a 100644 --- a/backend/tests/integration/parallel/__snapshots__/file.test.ts.snap +++ b/backend/tests/integration/parallel/__snapshots__/file.test.ts.snap @@ -4,7 +4,7 @@ exports[`/api/files/:uuid request succeeds on instrument file 1`] = ` { "checksum": "298688b011a511f8f0e9353371cf73ee86f60c89b0e02c6931d8c05542c64cdb", "createdAt": "2020-02-20T10:56:19.382Z", - "downloadUrl": "http://localhost:3000/api/download/product/38092c00-161d-4ca2-a29d-628cf8e960f6/20181115_mace-head_mira.nc", + "downloadUrl": "http://localhost:3000/api/download/product/38092c00-161d-4ca2-a29d-628cf8e960f6/38092c00-161d-4ca2-a29d-628cf8e960f6/20181115_mace-head_mira.nc", "dvasId": null, "dvasUpdatedAt": null, "errorLevel": null, diff --git a/backend/tests/integration/parallel/search.test.ts b/backend/tests/integration/parallel/search.test.ts index 542280355..8aa9cc875 100644 --- a/backend/tests/integration/parallel/search.test.ts +++ b/backend/tests/integration/parallel/search.test.ts @@ -223,7 +223,7 @@ describe("/api/files", () => { }; const res = await axios.get(url, payload); return expect(res.data[0]).toMatchObject({ - s3path: "/cloudnet-product/legacy/20090716_bucharest_classification.nc", + s3path: "/cloudnet-product/3bb32746-faf0-4057-9076-ed2e698dcf36/legacy/20090716_bucharest_classification.nc", }); }); diff --git a/backend/tests/integration/sequential/file.test.ts b/backend/tests/integration/sequential/file.test.ts index 8b7121b24..a821979cc 100644 --- a/backend/tests/integration/sequential/file.test.ts +++ b/backend/tests/integration/sequential/file.test.ts @@ -731,12 +731,14 @@ describe("DELETE /api/files/", () => { volatile: false, pid: "https://hdl.handle.net/123/pid1", }); - const file2 = await putDummyFile({ - product: "radar", - volatile: false, - pid: "https://hdl.handle.net/123/pid2", - version: true, - }); + const file2 = await putDummyFile( + { + product: "radar", + volatile: false, + pid: "https://hdl.handle.net/123/pid2", + }, + true, + ); await expect(fileRepo.findOneBy({ uuid: file1.uuid })).resolves.toBeTruthy(); await expect(fileRepo.findOneBy({ uuid: file2.uuid })).resolves.toBeTruthy(); expect(await searchFileRepo.existsBy({ uuid: file1.uuid })).toBeFalsy(); @@ -744,8 +746,8 @@ describe("DELETE /api/files/", () => { await expect(deleteFile(file2.uuid, false, false, "Blah")).resolves.toMatchObject({ status: 200 }); await expect(fileRepo.findOneBy({ uuid: file1.uuid })).resolves.toBeTruthy(); await expect(fileRepo.findOneBy({ uuid: file2.uuid })).resolves.toBeTruthy(); - expect(await searchFileRepo.existsBy({ uuid: file1.uuid })).toBeTruthy(); expect(await searchFileRepo.existsBy({ uuid: file2.uuid })).toBeFalsy(); + expect(await searchFileRepo.existsBy({ uuid: file1.uuid })).toBeTruthy(); }); it("test returning to a legacy file in delete", async () => { @@ -755,12 +757,14 @@ describe("DELETE /api/files/", () => { pid: "https://hdl.handle.net/123/pid1", legacy: true, }); - const file2 = await putDummyFile({ - product: "radar", - volatile: false, - pid: "https://hdl.handle.net/123/pid2", - version: true, - }); + const file2 = await putDummyFile( + { + product: "radar", + volatile: false, + pid: "https://hdl.handle.net/123/pid2", + }, + true, + ); await expect(fileRepo.findOneBy({ uuid: file1.uuid })).resolves.toBeTruthy(); await expect(fileRepo.findOneBy({ uuid: file2.uuid })).resolves.toBeTruthy(); expect(await searchFileRepo.existsBy({ uuid: file1.uuid })).toBeFalsy(); @@ -777,11 +781,13 @@ describe("DELETE /api/files/", () => { volatile: false, pid: "https://hdl.handle.net/123/pid1", }); - const file2 = await putDummyFile({ - volatile: false, - pid: "https://hdl.handle.net/123/pid2", - version: true, - }); + const file2 = await putDummyFile( + { + volatile: false, + pid: "https://hdl.handle.net/123/pid2", + }, + true, + ); await expect(deleteFile(file1.uuid, false, false, "Blah")).resolves.toMatchObject({ status: 200 }); expect(await searchFileRepo.existsBy({ uuid: file1.uuid })).toBeFalsy(); expect(await searchFileRepo.existsBy({ uuid: file2.uuid })).toBeTruthy(); @@ -792,16 +798,20 @@ describe("DELETE /api/files/", () => { volatile: false, pid: "https://hdl.handle.net/123/pid1", }); - const file2 = await putDummyFile({ - volatile: false, - pid: "https://hdl.handle.net/123/pid2", - version: true, - }); - const file3 = await putDummyFile({ - volatile: false, - pid: "https://hdl.handle.net/123/pid3", - version: true, - }); + const file2 = await putDummyFile( + { + volatile: false, + pid: "https://hdl.handle.net/123/pid2", + }, + true, + ); + const file3 = await putDummyFile( + { + volatile: false, + pid: "https://hdl.handle.net/123/pid3", + }, + true, + ); await expect(deleteFile(file2.uuid, false, false, "Blah")).resolves.toMatchObject({ status: 200 }); await expect(deleteFile(file3.uuid, false, false, "Blah")).resolves.toMatchObject({ status: 200 }); expect(await searchFileRepo.existsBy({ uuid: file1.uuid })).toBeTruthy(); @@ -814,24 +824,28 @@ describe("DELETE /api/files/", () => { volatile: false, pid: "https://hdl.handle.net/123/pid1", }); - const file2 = await putDummyFile({ - volatile: false, - pid: "https://hdl.handle.net/123/pid2", - version: true, - }); + const file2 = await putDummyFile( + { + volatile: false, + pid: "https://hdl.handle.net/123/pid2", + }, + true, + ); const catFile1 = await putDummyFile({ product: "categorize", volatile: false, sourceFileIds: [file1.uuid], pid: "https://hdl.handle.net/123/pid3", }); - const catFile2 = await putDummyFile({ - product: "categorize", - volatile: false, - sourceFileIds: [file2.uuid], - pid: "https://hdl.handle.net/123/pid4", - version: true, - }); + const catFile2 = await putDummyFile( + { + product: "categorize", + volatile: false, + sourceFileIds: [file2.uuid], + pid: "https://hdl.handle.net/123/pid4", + }, + true, + ); expect(await searchFileRepo.existsBy({ uuid: file2.uuid })).toBeTruthy(); expect(await searchFileRepo.existsBy({ uuid: catFile2.uuid })).toBeTruthy(); @@ -859,18 +873,20 @@ describe("DELETE /api/files/", () => { volatile: boolean; sourceFileIds: string[]; pid: string; - version: boolean; legacy: boolean; }> = {}, + version: boolean = false, ) { - const statusCode = options.version ? 200 : 201; + const statusCode = version ? 200 : 201; const { product = "radar", volatile = true, pid = null, legacy = false } = options; const fileFix = legacy ? "legacy/" : ""; + const uuid = uuidGen.v4(); + const filename = `20181115_mace-head_${product}.nc`; const file = { ...volatileFile, ...options, - ...{ uuid: uuidGen.v4(), product, volatile }, - s3key: `${fileFix}20181115_mace-head_${product}.nc`, + ...{ uuid, product, volatile, filename }, + s3key: `${uuid}/${fileFix}${filename}`, checksum: generateHash(), }; if (pid) file.pid = pid;