prepare new S3 files key format

server/modules/record/manager/fileManager.js

@@ -1,4 +1,4 @@
-import { Objects, Promises } from '@openforis/arena-core'
+import { Objects } from '@openforis/arena-core'
 
 import { ENV } from '@core/processUtils'
 import * as RecordFile from '@core/record/recordFile'
@@ -119,13 +119,12 @@ export const moveFilesToNewStorageIfNecessary = async ({ surveyId }, client = db
   logger.debug(`Survey ${surveyId}: started moving ${fileUuids.length} files from DB to new storage (${storageType})`)
 
   await client.tx(async (tx) => {
-    await Promises.each(fileUuids, async (fileUuid) => {
+    for await (const fileUuid of fileUuids) {
       const file = await FileRepository.fetchFileAndContentByUuid(surveyId, fileUuid, tx)
-
-      const contentStoreFunction = contentStoreFunctionByStorageType[storageType]
       const content = RecordFile.getContent(file)
+      const contentStoreFunction = contentStoreFunctionByStorageType[storageType]
       await contentStoreFunction({ surveyId, fileUuid, content })
-    })
+    }
     logger.debug(`Files moved from DB; clearing 'content' column in DB 'file' table`)
     await FileRepository.clearAllSurveyFilesContent({ surveyId }, tx)
   })

server/modules/record/repository/fileRepositoryS3Bucket.js

@@ -2,6 +2,7 @@ import {
   DeleteObjectCommand,
   GetObjectCommand,
   HeadBucketCommand,
+  HeadObjectCommand,
   PutObjectCommand,
   S3Client,
 } from '@aws-sdk/client-s3'
@@ -18,16 +19,21 @@ const s3Client = new S3Client({
   region: ProcessUtils.ENV.fileStorageAwsS3BucketRegion,
 })
 
-const getFileKey = ({ surveyId, fileUuid }) => `${surveyId}_${fileUuid}`
+const getFileKey = ({ surveyId, fileUuid }) => `surveys/${surveyId}/files/${fileUuid}`
 
-const createCommandParams = ({ surveyId, fileUuid }) => ({
-  Bucket,
-  Key: getFileKey({ surveyId, fileUuid }),
-})
+const createCommandParams = ({ surveyId, fileUuid }) => {
+  const Key = getFileKey({ surveyId, fileUuid })
+  return { Bucket, Key }
+}
+
+const fileExists = async (Key) => {
+  const command = new HeadObjectCommand({ Bucket, Key })
+  const data = await s3Client.send(command)
+  return data.$metadata.httpStatusCode === 200
+}
 
 export const checkCanAccessS3Bucket = async () => {
   const command = new HeadBucketCommand({ Bucket })
-
   try {
     await s3Client.send(command)
     return true
@@ -57,3 +63,21 @@ export const deleteFiles = async ({ surveyId, fileUuids }) =>
       return s3Client.send(command)
     })
   )
+
+const getOldFileKey = ({ surveyId, fileUuid }) => `${surveyId}_${fileUuid}`
+
+const createOldKeyCommandParams = ({ surveyId, fileUuid }) => {
+  const Key = getOldFileKey({ surveyId, fileUuid })
+  return { Bucket, Key }
+}
+
+export const oldKeyFileExists = async ({ surveyId, fileUuid }) => {
+  const Key = getOldFileKey({ surveyId, fileUuid })
+  return fileExists(Key)
+}
+
+export const getOldKeyFileContentAsStream = async ({ surveyId, fileUuid }) => {
+  const command = new GetObjectCommand(createOldKeyCommandParams({ surveyId, fileUuid }))
+  const response = await s3Client.send(command)
+  return response.Body
+}

server/modules/record/service/fileService.js

@@ -1,5 +1,3 @@
-import { Promises } from '@openforis/arena-core'
-
 import * as Log from '@server/log/log'
 import * as SurveyRepository from '@server/modules/survey/repository/surveyRepository'
 
@@ -23,15 +21,15 @@ export const checkFilesStorage = async () => {
   const surveyIds = await SurveyRepository.fetchAllSurveyIds()
   let allSurveysFilesMoved = false
   let errorsFound = false
-  await Promises.each(surveyIds, async (surveyId) => {
+  for await (const surveyId of surveyIds) {
     try {
       const surveyFilesMoved = await FileManager.moveFilesToNewStorageIfNecessary({ surveyId })
       allSurveysFilesMoved = allSurveysFilesMoved || surveyFilesMoved
     } catch (error) {
       errorsFound = true
       logger.error(`Error moving files for survey ${surveyId}`, error)
     }
-  })
+  }
   if (errorsFound) {
     logger.error(`There were errors moving survey files to the new storage`)
   } else if (allSurveysFilesMoved) {