Survey labels import (#3131)

* survey labels import (WIP)

* labels import job

* fixed labels import job server side

* added FileUploadDialog

* code cleanup

* reset survey defs on job complete

* reset survey defs on job complete

* fixing tests

* survey labels export

---------

Co-authored-by: Stefano Ricci <[email protected]>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

core/i18n/resources/en.js

@@ -5,6 +5,7 @@ export const enTranslation = {
   common: {
     active: 'Active',
     add: 'Add',
+    advancedFunctions: 'Advanced functions',
     and: 'and',
     appName: 'Arena',
     appNameFull: '$t(common.openForis) Arena',
@@ -394,7 +395,6 @@ Thank you and enjoy **$t(common.appNameFull)**!`,
     },
     surveyDeleted: 'Survey {{surveyName}} has been deleted',
     surveyInfo: {
-      advancedFunctions: 'Advanced functions',
       confirmDeleteCycleHeader: 'Delete this cycle?',
       confirmDeleteCycle: `Are you sure you want to delete the cycle {{cycle}}?\n\n$t(common.cantUndoWarning)\n\n
 If there are records associated to this cycle, they will be deleted.`,
@@ -1317,6 +1317,8 @@ $t(surveyForm.formEntryActions.confirmPromote)`,
       analysis: 'Analysis',
     },
     confirmNodeDelete: 'Are you sure you want to delete this item?',
+    exportLabels: 'Export labels to CSV',
+    importLabels: 'Import labels from CSV',
   },
 
   taxonomy: {
@@ -1552,6 +1554,11 @@ Levels will be renamed into level_1, level_2... level_N and an extra 'area' prop
       cycleDateEndMandatoryExceptForLastCycle: 'Cycle end date is mandatory for all but the last cycle',
     },
 
+    surveyLabelsImport: {
+      invalidHeaders: 'Invalid columns: {{invalidHeaders}}',
+      cannotFindNodeDef: "Cannot find attribute or entity definition with name '{{name}}'",
+    },
+
     taxonomyEdit: {
       codeChangedAfterPublishing: `Published code has changed: '{{oldCode}}' => '{{newCode}}'`,
       codeDuplicate: 'Duplicate code {{value}}; $t(validationErrors.rowsDuplicate)',
@@ -1648,6 +1655,7 @@ Levels will be renamed into level_1, level_2... level_N and an extra 'area' prop
     SurveyExportJob: 'Survey Export',
     SurveyIndexGeneratorJob: 'Survey Index Generator',
     SurveyInfoValidationJob: 'Survey Info Validation',
+    SurveyLabelsImportJob: 'Survey Labels Import',
     SurveyPropsPublishJob: 'Survey Props Publish',
     SurveyPublishJob: 'Survey Publish',
     SurveyPublishPerformJob: 'Survey Publish Perform',

core/survey/nodeDef.js

@@ -419,6 +419,9 @@ export const dissocTemporary = R.dissoc(keys.temporary)
 export const assocProp = ({ key, value }) =>
   isPropAdvanced(key) ? mergePropsAdvanced({ [key]: value }) : mergeProps({ [key]: value })
 export const assocCycles = (cycles) => assocProp({ key: propKeys.cycles, value: cycles })
+export const assocLabels = (labels) => assocProp({ key: propKeys.labels, value: labels })
+export const assocDescriptions = (descriptions) => assocProp({ key: propKeys.descriptions, value: descriptions })
+
 export const dissocEnumerate = ObjectUtils.dissocProp(propKeys.enumerate)
 export const cloneIntoEntityDef =
   ({ nodeDefParent, clonedNodeDefName }) =>

server/job/jobCreator.js

@@ -12,6 +12,7 @@ import ExportCsvDataJob from '@server/modules/survey/service/export/exportCsvDat
 import RecordsCloneJob from '@server/modules/record/service/recordsCloneJob'
 import SurveyCloneJob from '@server/modules/survey/service/clone/surveyCloneJob'
 import SurveyExportJob from '@server/modules/survey/service/surveyExport/surveyExportJob'
+import SurveyLabelsImportJob from '@server/modules/survey/service/surveyLabelsImportJob'
 import SurveyPublishJob from '@server/modules/survey/service/publish/surveyPublishJob'
 import SurveysRdbRefreshJob from '@server/modules/surveyRdb/service/SurveysRdbRefreshJob'
 import SurveyUnpublishJob from '@server/modules/survey/service/unpublish/surveyUnpublishJob'
@@ -30,6 +31,7 @@ const jobClasses = [
   RecordsCloneJob,
   SurveyCloneJob,
   SurveyExportJob,
+  SurveyLabelsImportJob,
   SurveyPublishJob,
   SurveysRdbRefreshJob,
   SurveyUnpublishJob,

server/modules/survey/api/surveyApi.js

@@ -189,6 +189,20 @@ export const init = (app) => {
     }
   })
 
+  app.get('/survey/:surveyId/labels', AuthMiddleware.requireSurveyViewPermission, async (req, res, next) => {
+    try {
+      const { surveyId } = Request.getParams(req)
+
+      const survey = await SurveyService.fetchSurveyById({ surveyId, draft: true })
+      const fileName = ExportFileNameGenerator.generate({ survey, fileType: 'Labels' })
+      Response.setContentTypeFile({ res, fileName, contentType: Response.contentTypes.csv })
+
+      await SurveyService.exportLabels({ surveyId, outputStream: res })
+    } catch (error) {
+      next(error)
+    }
+  })
+
   // ==== UPDATE
 
   app.put('/survey/:surveyId/info', AuthMiddleware.requireSurveyEditPermission, async (req, res, next) => {
@@ -224,6 +238,20 @@ export const init = (app) => {
     res.json({ job: JobUtils.jobToJSON(job) })
   })
 
+  app.put('/survey/:surveyId/labels', AuthMiddleware.requireSurveyEditPermission, async (req, res, next) => {
+    try {
+      const user = Request.getUser(req)
+      const filePath = Request.getFilePath(req)
+      const { surveyId } = Request.getParams(req)
+
+      const job = SurveyService.startLabelsImportJob({ user, surveyId, filePath })
+
+      res.json({ job: JobUtils.jobToJSON(job) })
+    } catch (error) {
+      next(error)
+    }
+  })
+
   // ==== DELETE
 
   app.delete('/survey/:surveyId', AuthMiddleware.requireSurveyEditPermission, async (req, res, next) => {

server/modules/survey/service/surveyLabelsExport.js

@@ -0,0 +1,35 @@
+import * as Survey from '@core/survey/survey'
+import * as NodeDef from '@core/survey/nodeDef'
+
+import * as CSVWriter from '@server/utils/file/csvWriter'
+import * as SurveyManager from '@server/modules/survey/manager/surveyManager'
+
+const exportLabels = async ({ surveyId, outputStream }) => {
+  const survey = await SurveyManager.fetchSurveyAndNodeDefsBySurveyId({ surveyId, draft: true, includeAnalysis: false })
+  const languages = Survey.getLanguages(Survey.getSurveyInfo(survey))
+  const items = []
+  Survey.visitDescendantsAndSelf({
+    nodeDef: Survey.getNodeDefRoot(survey),
+    visitorFn: (nodeDef) => {
+      items.push({
+        name: NodeDef.getName(nodeDef),
+        // labels
+        ...languages.reduce(
+          (labelsAcc, lang) => ({ ...labelsAcc, [`label_${lang}`]: NodeDef.getLabel(nodeDef, lang) }),
+          {}
+        ),
+        // description
+        ...languages.reduce(
+          (labelsAcc, lang) => ({ ...labelsAcc, [`description_${lang}`]: NodeDef.getDescription(lang)(nodeDef) }),
+          {}
+        ),
+      })
+    },
+  })(survey)
+
+  await CSVWriter.writeItemsToStream({ outputStream, items, options: { removeNewLines: false } })
+}
+
+export const SurveyLabelsExport = {
+  exportLabels,
+}

server/modules/survey/service/surveyLabelsExportModel.js

@@ -0,0 +1,11 @@
+const labelColumnPrefix = 'label_'
+const descriptionColumnPrefix = 'description_'
+const getLabelColumn = (langCode) => `${labelColumnPrefix}${langCode}`
+const getDescriptionColumn = (langCode) => `${descriptionColumnPrefix}${langCode}`
+
+export const SurveyLabelsExportModel = {
+  labelColumnPrefix,
+  descriptionColumnPrefix,
+  getLabelColumn,
+  getDescriptionColumn,
+}

server/modules/survey/service/surveyLabelsImportJob.js

@@ -0,0 +1,123 @@
+import * as A from '@core/arena'
+import * as Survey from '@core/survey/survey'
+import * as NodeDef from '@core/survey/nodeDef'
+import * as StringUtils from '@core/stringUtils'
+
+import Job from '@server/job/job'
+import * as CSVReader from '@server/utils/file/csvReader'
+import * as SurveyManager from '@server/modules/survey/manager/surveyManager'
+import * as NodeDefManager from '@server/modules/nodeDef/manager/nodeDefManager'
+import { SurveyLabelsExportModel } from './surveyLabelsExportModel'
+
+const errorPrefix = `validationErrors.surveyLabelsImport.`
+
+const extractLabels = ({ row, langCodes, columnPrefix }) =>
+  langCodes.reduce((acc, langCode) => {
+    const value = row[`${columnPrefix}${langCode}`]
+    const label = StringUtils.trim(value)
+    if (StringUtils.isNotBlank(label)) {
+      acc[langCode] = label
+    }
+    return acc
+  }, {})
+
+export default class SurveyLabelsImportJob extends Job {
+  constructor(params) {
+    super(SurveyLabelsImportJob.type, params)
+    this.validateHeaders = this.validateHeaders.bind(this)
+  }
+
+  async execute() {
+    const { context, tx } = this
+    const { filePath, surveyId } = context
+    const survey = await SurveyManager.fetchSurveyAndNodeDefsBySurveyId({ surveyId, draft: true }, tx)
+    this.setContext({ survey })
+    const langCodes = Survey.getLanguages(Survey.getSurveyInfo(survey))
+
+    const nodeDefsUpdated = []
+
+    this.csvReader = CSVReader.createReaderFromFile(
+      filePath,
+      this.validateHeaders,
+      async (row) => {
+        const nodeDef = await this.getNodeDef({ survey, row })
+        if (!nodeDef) return
+
+        const labels = extractLabels({ langCodes, row, columnPrefix: SurveyLabelsExportModel.labelColumnPrefix })
+        const descriptions = extractLabels({
+          langCodes,
+          row,
+          columnPrefix: SurveyLabelsExportModel.descriptionColumnPrefix,
+        })
+
+        const nodeDefUpdated = A.pipe(NodeDef.assocLabels(labels), NodeDef.assocDescriptions(descriptions))(nodeDef)
+        nodeDefsUpdated.push(nodeDefUpdated)
+
+        this.incrementProcessedItems()
+      },
+      (total) => (this.total = total)
+    )
+
+    await this.csvReader.start()
+
+    if (nodeDefsUpdated.length > 0) {
+      await NodeDefManager.updateNodeDefPropsInBatch(
+        {
+          surveyId,
+          nodeDefs: nodeDefsUpdated.map((nodeDefUpdated) => ({
+            nodeDefUuid: NodeDef.getUuid(nodeDefUpdated),
+            props: NodeDef.getProps(nodeDefUpdated),
+          })),
+        },
+        tx
+      )
+    }
+  }
+
+  async validateHeaders(headers) {
+    const { context } = this
+    const { survey } = context
+    const langCodes = Survey.getLanguages(Survey.getSurveyInfo(survey))
+
+    const fixedHeaders = ['name']
+    const dynamicHeaders = []
+    langCodes.forEach((langCode) => {
+      dynamicHeaders.push(
+        SurveyLabelsExportModel.getLabelColumn(langCode),
+        SurveyLabelsExportModel.getDescriptionColumn(langCode)
+      )
+    })
+    const validHeaders = [...fixedHeaders, ...dynamicHeaders]
+    const invalidHeaders = headers.filter((header) => !validHeaders.includes(header)).join(', ')
+    if (invalidHeaders) {
+      await this.addErrorAndStopCsvReader('invalidHeaders', { invalidHeaders })
+    }
+  }
+
+  async getNodeDef({ row }) {
+    const { context } = this
+    const { survey } = context
+    const { name } = row
+    let nodeDef = null
+    if (name) {
+      nodeDef = Survey.getNodeDefByName(name)(survey)
+    }
+    if (!nodeDef) {
+      await this.addErrorAndStopCsvReader('cannotFindNodeDef', { name })
+    }
+    return nodeDef
+  }
+
+  async addErrorAndStopCsvReader(key, params) {
+    this.addError({
+      error: {
+        valid: false,
+        errors: [{ key: `${errorPrefix}${key}`, params }],
+      },
+    })
+    this.csvReader.cancel()
+    await this.setStatusFailed()
+  }
+}
+
+SurveyLabelsImportJob.type = 'SurveyLabelsImportJob'

server/modules/survey/service/surveyService.js

@@ -10,6 +10,8 @@ import SurveyExportJob from './surveyExport/surveyExportJob'
 import SurveyPublishJob from './publish/surveyPublishJob'
 import SurveyUnpublishJob from './unpublish/surveyUnpublishJob'
 import { SchemaSummary } from './schemaSummary'
+import SurveyLabelsImportJob from './surveyLabelsImportJob'
+import { SurveyLabelsExport } from './surveyLabelsExport'
 
 // JOBS
 export const startPublishJob = (user, surveyId) => {
@@ -76,12 +78,27 @@ export const startExportCsvDataJob = ({
 export const exportSchemaSummary = async ({ surveyId, cycle, outputStream }) =>
   SchemaSummary.exportSchemaSummary({ surveyId, cycle, outputStream })
 
+export const exportLabels = async ({ surveyId, outputStream }) =>
+  SurveyLabelsExport.exportLabels({ surveyId, outputStream })
+
 export const deleteSurvey = async (surveyId) => {
   RecordsUpdateThreadService.clearSurveyDataFromThread({ surveyId })
 
   await SurveyManager.deleteSurvey(surveyId)
 }
 
+export const startLabelsImportJob = ({ user, surveyId, filePath }) => {
+  const job = new SurveyLabelsImportJob({
+    user,
+    surveyId,
+    filePath,
+  })
+
+  JobManager.executeJobThread(job)
+
+  return job
+}
+
 export const {
   // CREATE
   insertSurvey,

test/e2e/tests/surveySchemaSummary.js

@@ -47,6 +47,9 @@ export default () =>
     test('export schema summary', async () => {
       const [download] = await Promise.all([
         page.waitForEvent('download'),
+
+        page.click(getSelector(TestId.surveyForm.advancedFunctionBtn, 'button')),
+
         page.click(getSelector(TestId.surveyForm.schemaSummary, 'button')),
       ])
 

webapp/components/buttons/ButtonMenu.js

@@ -39,7 +39,7 @@ export const ButtonMenu = (props) => {
 
       <Menu anchorEl={anchorEl} className={menuClassName} open={open} onClose={closeMenu}>
         {items.map((item) => (
-          <MenuItem key={item.key} onClick={onItemClick(item)}>
+          <MenuItem key={item.key} className="button-menu__item" onClick={onItemClick(item)}>
             {item.content ? (
               item.content
             ) : (

webapp/components/buttons/ButtonMenu.scss

@@ -7,3 +7,13 @@
     transform: rotate(180deg);
   }
 }
+
+.MuiMenuItem-root.button-menu__item {
+  padding: 0;
+
+  button {
+    width: 100%;
+    text-align: left;
+    padding: 16px 24px;
+  }
+}

webapp/components/form/index.js

@@ -3,10 +3,20 @@ import Checkbox from './checkbox'
 import Dropdown from './Dropdown'
 import EmailInput from './EmailInput'
 import LanguageDropdown from './languageDropdown'
+import OpenFileUploadDialogButton from './openFileUploadDialogButton'
 import Radiobox from './radiobox'
 import UploadButton from './uploadButton'
 
-export { ButtonGroup, Checkbox, Dropdown, EmailInput, LanguageDropdown, Radiobox, UploadButton }
+export {
+  ButtonGroup,
+  Checkbox,
+  Dropdown,
+  EmailInput,
+  LanguageDropdown,
+  OpenFileUploadDialogButton,
+  Radiobox,
+  UploadButton,
+}
 
 export { PasswordInput } from './PasswordInput'
 export { PasswordStrengthChecker } from './PasswordStrengthChecker'