Files import: adding logs; replacing files marked as deleted (#3133)

Co-authored-by: Stefano Ricci <[email protected]>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

server/modules/arenaImport/service/arenaImport/jobs/filesImportJob.js

@@ -18,6 +18,10 @@ export default class FilesImportJob extends Job {
 
     const filesSummaries = await ArenaSurveyFileZip.getFilesSummaries(arenaSurveyFileZip)
     if (filesSummaries && filesSummaries.length > 0) {
+      const filesUuids = filesSummaries.map(RecordFile.getUuid)
+      this.logDebug('file UUIDs in zip file', filesUuids)
+      this.logDebug('file UUIDs found in records', this.context.recordsFileUuids)
+
       await this.checkFilesNotExceedingAvailableQuota(filesSummaries)
 
       this.total = filesSummaries.length
@@ -50,13 +54,25 @@ export default class FilesImportJob extends Job {
   }
 
   async persistFile(file) {
-    const { surveyId } = this.context
+    const { context, tx } = this
+    const { surveyId } = context
     const fileUuid = RecordFile.getUuid(file)
+    const fileProps = RecordFile.getProps(file)
+    this.logDebug(`persisting file ${fileUuid}`)
     const existingFileSummary = await FileService.fetchFileSummaryByUuid(surveyId, fileUuid, this.tx)
     if (existingFileSummary) {
-      await FileService.updateFileProps(surveyId, fileUuid, RecordFile.getProps(file), this.tx)
+      this.logDebug(`file already existing`)
+      if (RecordFile.isDeleted(existingFileSummary)) {
+        this.logDebug(`file previously marked as deleted: delete permanently and insert a new one`)
+        await FileService.deleteFileByUuid({ surveyId, fileUuid }, tx)
+        await FileService.insertFile(surveyId, file, tx)
+      } else {
+        this.logDebug('updating props')
+        await FileService.updateFileProps(surveyId, fileUuid, fileProps, tx)
+      }
     } else {
-      await FileService.insertFile(surveyId, file, this.tx)
+      this.logDebug(`file not existing: inserting new file`, fileProps)
+      await FileService.insertFile(surveyId, file, tx)
     }
   }
 

server/modules/mobile/service/arenaMobileDataImport/arenaMobileDataImportJob.js

@@ -7,6 +7,7 @@ import RecordsImportJob from './jobs/recordsImportJob'
 import FilesImportJob from '../../../arenaImport/service/arenaImport/jobs/filesImportJob'
 import { RecordsUpdateThreadService } from '@server/modules/record/service/update/surveyRecordsThreadService'
 import { RecordsUpdateThreadMessageTypes } from '@server/modules/record/service/update/thread/recordsThreadMessageTypes'
+import * as SurveyService from '@server/modules/survey/service/surveyService'
 
 export default class ArenaMobileDataImportJob extends Job {
   /**
@@ -24,12 +25,16 @@ export default class ArenaMobileDataImportJob extends Job {
   async onStart() {
     await super.onStart()
 
-    const { filePath } = this.context
+    const { context, tx } = this
+    const { filePath, surveyId } = context
 
     const arenaSurveyFileZip = new FileZip(filePath)
     await arenaSurveyFileZip.init()
 
     this.setContext({ arenaSurveyFileZip })
+
+    const survey = await SurveyService.fetchSurveyAndNodeDefsAndRefDataBySurveyId({ surveyId, advanced: true }, tx)
+    this.setContext({ survey })
   }
 
   async beforeSuccess() {

server/modules/mobile/service/arenaMobileDataImport/jobs/recordsImportJob.js

@@ -13,7 +13,6 @@ import * as PromiseUtils from '@core/promiseUtils'
 import * as ArenaSurveyFileZip from '@server/modules/arenaImport/service/arenaImport/model/arenaSurveyFileZip'
 import DataImportBaseJob from '@server/modules/dataImport/service/DataImportJob/DataImportBaseJob'
 import * as RecordManager from '@server/modules/record/manager/recordManager'
-import * as SurveyService from '@server/modules/survey/service/surveyService'
 import * as UserService from '@server/modules/user/service/userService'
 
 export default class RecordsImportJob extends DataImportBaseJob {
@@ -26,16 +25,13 @@ export default class RecordsImportJob extends DataImportBaseJob {
   async execute() {
     await super.execute()
 
-    const { context, tx } = this
-    const { arenaSurveyFileZip, surveyId } = context
+    const { context } = this
+    const { arenaSurveyFileZip } = context
 
     const recordSummaries = await ArenaSurveyFileZip.getRecords(arenaSurveyFileZip)
     this.total = recordSummaries.length
 
-    if (this.total == 0) return
-
-    const survey = await SurveyService.fetchSurveyAndNodeDefsAndRefDataBySurveyId({ surveyId, advanced: true }, tx)
-    this.setContext({ survey })
+    if (this.total === 0) return
 
     // import records sequentially
     await PromiseUtils.each(recordSummaries, async (recordSummary) => {

server/modules/record/manager/fileManager.js

@@ -111,6 +111,11 @@ export const moveFilesToNewStorageIfNecessary = async ({ surveyId }, client = db
   return true
 }
 
+export const deleteFileByUuid = async ({ surveyId, fileUuid }, client = db) => {
+  await FileRepository.deleteFileByUuid(surveyId, fileUuid, client)
+  // do not delete content if not in DB: deletion out of transaction
+}
+
 export const deleteFilesByRecordUuids = async (surveyId, recordUuids, client = db) => {
   const storageType = getFileContentStorageType()
   const deleteFn = contentDeleteFunctionByStorageType[storageType]

server/modules/record/service/fileService.js

@@ -40,6 +40,7 @@ export const checkFilesStorage = async () => {
   }
 }
 
+// eslint-disable-next-line no-unused-vars
 const getSurveyFilesTotalSpace = async ({ surveyId }) => {
   return 10 * Math.pow(1024, 3) // TODO make it configurable, fixed to 10 GB per survey now
 }
@@ -65,4 +66,6 @@ export const {
   fetchFileSummariesBySurveyId,
   // UPDATE
   updateFileProps,
+  // DELETE
+  deleteFileByUuid,
 } = FileManager