Merge branch 'fix/charts' of github.com:openforis/arena into fix/charts

common/webSocket/webSocketEvents.js

@@ -12,6 +12,8 @@ export const WebSocketEvents = {
   nodesUpdateCompleted: 'nodesUpdateCompleted',
   nodeValidationsUpdate: 'nodeValidationsUpdate',
   recordDelete: 'recordDelete',
+  surveyUpdate: 'surveyUpdate',
+
   error: 'threadError',
   recordSessionExpired: 'recordSessionExpired',
   applicationError: 'applicationError',

core/i18n/resources/en.js

@@ -5,6 +5,7 @@ export const enTranslation = {
   common: {
     active: 'Active',
     add: 'Add',
+    advancedFunctions: 'Advanced functions',
     and: 'and',
     appName: 'Arena',
     appNameFull: '$t(common.openForis) Arena',
@@ -394,7 +395,6 @@ Thank you and enjoy **$t(common.appNameFull)**!`,
     },
     surveyDeleted: 'Survey {{surveyName}} has been deleted',
     surveyInfo: {
-      advancedFunctions: 'Advanced functions',
       confirmDeleteCycleHeader: 'Delete this cycle?',
       confirmDeleteCycle: `Are you sure you want to delete the cycle {{cycle}}?\n\n$t(common.cantUndoWarning)\n\n
 If there are records associated to this cycle, they will be deleted.`,
@@ -768,6 +768,7 @@ Are you sure you want to continue?`,
   },
 
   surveysView: {
+    datePublished: 'Date published',
     filter: 'Filter',
     filterPlaceholder: 'Filter by name, label or owner',
     noSurveysMatchingFilter: 'No surveys matching the specified filter',
@@ -1316,6 +1317,8 @@ $t(surveyForm.formEntryActions.confirmPromote)`,
       analysis: 'Analysis',
     },
     confirmNodeDelete: 'Are you sure you want to delete this item?',
+    exportLabels: 'Export labels to CSV',
+    importLabels: 'Import labels from CSV',
   },
 
   taxonomy: {
@@ -1551,6 +1554,11 @@ Levels will be renamed into level_1, level_2... level_N and an extra 'area' prop
       cycleDateEndMandatoryExceptForLastCycle: 'Cycle end date is mandatory for all but the last cycle',
     },
 
+    surveyLabelsImport: {
+      invalidHeaders: 'Invalid columns: {{invalidHeaders}}',
+      cannotFindNodeDef: "Cannot find attribute or entity definition with name '{{name}}'",
+    },
+
     taxonomyEdit: {
       codeChangedAfterPublishing: `Published code has changed: '{{oldCode}}' => '{{newCode}}'`,
       codeDuplicate: 'Duplicate code {{value}}; $t(validationErrors.rowsDuplicate)',
@@ -1647,6 +1655,7 @@ Levels will be renamed into level_1, level_2... level_N and an extra 'area' prop
     SurveyExportJob: 'Survey Export',
     SurveyIndexGeneratorJob: 'Survey Index Generator',
     SurveyInfoValidationJob: 'Survey Info Validation',
+    SurveyLabelsImportJob: 'Survey Labels Import',
     SurveyPropsPublishJob: 'Survey Props Publish',
     SurveyPublishJob: 'Survey Publish',
     SurveyPublishPerformJob: 'Survey Publish Perform',

core/survey/_survey/surveyInfo.js

@@ -15,6 +15,7 @@ export const keys = {
   ownerName: 'ownerName',
   draft: 'draft',
   published: ObjectUtils.keys.published,
+  datePublished: 'datePublished',
   authGroups: 'authGroups',
   props: ObjectUtils.keys.props,
   rdbInitialized: 'rdbInitialized',
@@ -139,6 +140,8 @@ export const getDefaultCycleKey = (surveyInfo) => {
 
 export const { getDateCreated, getDateModified } = ObjectUtils
 
+export const getDatePublished = ObjectUtils.getDate(keys.datePublished)
+
 export const getCollectUri = ObjectUtils.getProp(keys.collectUri)
 
 export const getCollectReport = ObjectUtils.getProp(keys.collectReport, {})

core/survey/_survey/surveyNodeDefs.js

@@ -1,6 +1,8 @@
 import * as R from 'ramda'
 
-import { Surveys } from '@openforis/arena-core'
+import { Surveys, TraverseMethod } from '@openforis/arena-core'
+
+import Queue from '@core/queue'
 
 import * as PromiseUtils from '../../promiseUtils'
 import * as NodeDef from '../nodeDef'
@@ -9,7 +11,6 @@ import * as NodeDefValidations from '../nodeDefValidations'
 import * as Category from '../category'
 import * as SurveyInfo from './surveyInfo'
 import * as SurveyNodeDefsIndex from './surveyNodeDefsIndex'
-import Queue from '@core/queue'
 
 const nodeDefsKey = 'nodeDefs'
 
@@ -312,22 +313,15 @@ export const traverseHierarchyItemSync = (nodeDefItem, visitorFn, depth = 0) =>
 }
 
 export const visitDescendantsAndSelf =
-  ({ nodeDef = null, visitorFn }) =>
+  ({ nodeDef = null, visitorFn, traverseMethod = TraverseMethod.bfs }) =>
   (survey) => {
-    const queue = new Queue()
-
-    queue.enqueue(nodeDef || getNodeDefRoot(survey))
-
-    while (!queue.isEmpty()) {
-      const nodeDefCurrent = queue.dequeue()
-
-      visitorFn(nodeDefCurrent)
-
-      if (NodeDef.isEntity(nodeDefCurrent)) {
-        const childrenDefs = getNodeDefChildren(nodeDefCurrent)(survey)
-        queue.enqueueItems(childrenDefs)
-      }
-    }
+    const nodeDefToVisit = nodeDef ?? getNodeDefRoot(survey)
+    return Surveys.visitDescendantsAndSelfNodeDef({
+      survey,
+      nodeDef: nodeDefToVisit,
+      visitor: visitorFn,
+      traverseMethod,
+    })
   }
 
 export const findDescendants =

core/survey/_survey/surveySortKeys.js

@@ -3,6 +3,7 @@ import * as ObjectUtils from '@core/objectUtils'
 export const sortableKeys = {
   dateCreated: ObjectUtils.keys.dateCreated,
   dateModified: ObjectUtils.keys.dateModified,
+  datePublished: 'datePublished',
   name: ObjectUtils.keys.name,
   ownerName: 'owner_name',
   label: 'label',

core/survey/nodeDef.js

@@ -419,6 +419,9 @@ export const dissocTemporary = R.dissoc(keys.temporary)
 export const assocProp = ({ key, value }) =>
   isPropAdvanced(key) ? mergePropsAdvanced({ [key]: value }) : mergeProps({ [key]: value })
 export const assocCycles = (cycles) => assocProp({ key: propKeys.cycles, value: cycles })
+export const assocLabels = (labels) => assocProp({ key: propKeys.labels, value: labels })
+export const assocDescriptions = (descriptions) => assocProp({ key: propKeys.descriptions, value: descriptions })
+
 export const dissocEnumerate = ObjectUtils.dissocProp(propKeys.enumerate)
 export const cloneIntoEntityDef =
   ({ nodeDefParent, clonedNodeDefName }) =>

core/survey/survey.js

@@ -117,6 +117,7 @@ export const {
   getDefaultCycleKey,
   getDateCreated,
   getDateModified,
+  getDatePublished,
   isPublished,
   isDraft,
   isValid,

package.json

@@ -106,8 +106,8 @@
     "@mui/material": "^5.14.10",
     "@mui/x-data-grid": "^6.14.0",
     "@mui/x-date-pickers": "^6.14.0",
-    "@openforis/arena-core": "^0.0.172",
-    "@openforis/arena-server": "^0.1.29",
+    "@openforis/arena-core": "^0.0.175",
+    "@openforis/arena-server": "^0.1.30",
     "@sendgrid/mail": "^7.7.0",
     "@shopify/draggable": "^1.0.0-beta.8",
     "ace-builds": "^1.27.0",

server/job/job.js

@@ -421,6 +421,10 @@ export default class Job {
     this._logger.info(...msgs)
   }
 
+  logWarn(...msgs) {
+    this._logger.warn(...msgs)
+  }
+
   logError(...msgs) {
     this._logger.error(...msgs)
   }

server/job/jobCreator.js

@@ -12,6 +12,7 @@ import ExportCsvDataJob from '@server/modules/survey/service/export/exportCsvDat
 import RecordsCloneJob from '@server/modules/record/service/recordsCloneJob'
 import SurveyCloneJob from '@server/modules/survey/service/clone/surveyCloneJob'
 import SurveyExportJob from '@server/modules/survey/service/surveyExport/surveyExportJob'
+import SurveyLabelsImportJob from '@server/modules/survey/service/surveyLabelsImportJob'
 import SurveyPublishJob from '@server/modules/survey/service/publish/surveyPublishJob'
 import SurveysRdbRefreshJob from '@server/modules/surveyRdb/service/SurveysRdbRefreshJob'
 import SurveyUnpublishJob from '@server/modules/survey/service/unpublish/surveyUnpublishJob'
@@ -30,6 +31,7 @@ const jobClasses = [
   RecordsCloneJob,
   SurveyCloneJob,
   SurveyExportJob,
+  SurveyLabelsImportJob,
   SurveyPublishJob,
   SurveysRdbRefreshJob,
   SurveyUnpublishJob,

server/modules/analysis/service/rChain/index.js

@@ -13,6 +13,8 @@ import { TableChain } from '@common/model/db'
 import { Query } from '@common/model/query'
 import * as Chain from '@common/analysis/chain'
 
+import * as UserService from '@server/modules/user/service/userService'
+
 import * as SurveyManager from '@server/modules/survey/manager/surveyManager'
 import * as NodeDefManager from '@server/modules/nodeDef/manager/nodeDefManager'
 import * as SurveyRdbManager from '@server/modules/surveyRdb/manager/surveyRdbManager'
@@ -166,4 +168,5 @@ export const persistUserScripts = async ({ user, surveyId, chainUuid, filePath }
       }
     })
   })
+  await UserService.notifyActiveUsersAboutSurveyUpdate({ surveyId })
 }

server/modules/arenaImport/service/arenaImport/jobs/filesImportJob.js

@@ -18,13 +18,25 @@ export default class FilesImportJob extends Job {
 
     const filesSummaries = await ArenaSurveyFileZip.getFilesSummaries(arenaSurveyFileZip)
     if (filesSummaries && filesSummaries.length > 0) {
+      const filesUuids = filesSummaries.map(RecordFile.getUuid)
+      this.logDebug('file UUIDs in zip file', filesUuids)
+      this.logDebug('file UUIDs found in records', this.context.recordsFileUuids)
+
       await this.checkFilesNotExceedingAvailableQuota(filesSummaries)
 
       this.total = filesSummaries.length
       await PromiseUtils.each(filesSummaries, async (fileSummary) => {
         let file = { ...fileSummary }
         // load file content from a separate file
-        const fileContent = await ArenaSurveyFileZip.getFile(arenaSurveyFileZip, RecordFile.getUuid(fileSummary))
+        const fileUuid = RecordFile.getUuid(fileSummary)
+        const fileContent = await ArenaSurveyFileZip.getFile(arenaSurveyFileZip, fileUuid)
+        if (!fileContent) {
+          const fileName = RecordFile.getName(fileSummary)
+          throw new Error(`Missing content for file ${fileUuid} (${fileName})`)
+        }
+
+        this.checkFileUuidIsInRecords(fileUuid)
+
         file = RecordFile.assocContent(fileContent)(file)
 
         await this.persistFile(file)
@@ -34,13 +46,33 @@ export default class FilesImportJob extends Job {
     }
   }
 
+  checkFileUuidIsInRecords(fileUuid) {
+    const { recordsFileUuids } = this.context
+    if (recordsFileUuids && !recordsFileUuids.includes(fileUuid)) {
+      throw new Error(`File UUID ${fileUuid} not found in records`)
+    }
+  }
+
   async persistFile(file) {
-    const { surveyId } = this.context
-    const existingFileSummary = await FileService.fetchFileSummaryByUuid(surveyId, file.uuid, this.tx)
+    const { context, tx } = this
+    const { surveyId } = context
+    const fileUuid = RecordFile.getUuid(file)
+    const fileProps = RecordFile.getProps(file)
+    this.logDebug(`persisting file ${fileUuid}`)
+    const existingFileSummary = await FileService.fetchFileSummaryByUuid(surveyId, fileUuid, this.tx)
     if (existingFileSummary) {
-      await FileService.updateFileProps(surveyId, RecordFile.getUuid(file), RecordFile.getProps(file), this.tx)
+      this.logDebug(`file already existing`)
+      if (RecordFile.isDeleted(existingFileSummary)) {
+        this.logDebug(`file previously marked as deleted: delete permanently and insert a new one`)
+        await FileService.deleteFileByUuid({ surveyId, fileUuid }, tx)
+        await FileService.insertFile(surveyId, file, tx)
+      } else {
+        this.logDebug('updating props')
+        await FileService.updateFileProps(surveyId, fileUuid, fileProps, tx)
+      }
     } else {
-      await FileService.insertFile(surveyId, file, this.tx)
+      this.logDebug(`file not existing: inserting new file`, fileProps)
+      await FileService.insertFile(surveyId, file, tx)
     }
   }
 

server/modules/mobile/service/arenaMobileDataImport/arenaMobileDataImportJob.js

@@ -7,6 +7,7 @@ import RecordsImportJob from './jobs/recordsImportJob'
 import FilesImportJob from '../../../arenaImport/service/arenaImport/jobs/filesImportJob'
 import { RecordsUpdateThreadService } from '@server/modules/record/service/update/surveyRecordsThreadService'
 import { RecordsUpdateThreadMessageTypes } from '@server/modules/record/service/update/thread/recordsThreadMessageTypes'
+import * as SurveyService from '@server/modules/survey/service/surveyService'
 
 export default class ArenaMobileDataImportJob extends Job {
   /**
@@ -24,12 +25,16 @@ export default class ArenaMobileDataImportJob extends Job {
   async onStart() {
     await super.onStart()
 
-    const { filePath } = this.context
+    const { context, tx } = this
+    const { filePath, surveyId } = context
 
     const arenaSurveyFileZip = new FileZip(filePath)
     await arenaSurveyFileZip.init()
 
     this.setContext({ arenaSurveyFileZip })
+
+    const survey = await SurveyService.fetchSurveyAndNodeDefsAndRefDataBySurveyId({ surveyId, advanced: true }, tx)
+    this.setContext({ survey })
   }
 
   async beforeSuccess() {