Merge branch 'master' into feat/survey-user-extra-props

common/model/csvExport/CsvDataExportModel.js

@@ -186,7 +186,7 @@ export class CsvDataExportModel {
 
   _extractAttributeDefsColumns(nodeDefContext) {
     const { cycle, options } = this
-    const { includeAnalysis, includeFiles, includeReadOnlyAttributes } = options
+    const { includeAnalysis, includeFileAttributeDefs, includeFiles, includeReadOnlyAttributes } = options
 
     let descendantDefs = NodeDef.isEntity(nodeDefContext)
       ? Survey.getNodeDefDescendantAttributesInSingleEntities({
@@ -199,7 +199,7 @@ export class CsvDataExportModel {
 
     descendantDefs = descendantDefs.filter(
       (nodeDef) =>
-        (includeFiles || !NodeDef.isFile(nodeDef)) &&
+        (includeFileAttributeDefs || includeFiles || !NodeDef.isFile(nodeDef)) &&
         (includeReadOnlyAttributes || !NodeDef.isReadOnly(nodeDef) || NodeDef.isKey(nodeDef))
     )
 

common/model/db/views/dataNodeDef/view.js

@@ -48,6 +48,10 @@ export default class ViewDataNodeDef extends TableDataNodeDef {
     return this.columnNodeDefUuid.nameFull
   }
 
+  get columnUuidName() {
+    return this.columnNodeDefUuid.name
+  }
+
   get columnIdName() {
     const { nodeDef } = this.tableData
     return `_${NodeDef.getName(nodeDef)}_${TableDataNodeDef.columnSet.id}`
@@ -118,6 +122,10 @@ export default class ViewDataNodeDef extends TableDataNodeDef {
     return this.columnNodeDefs.flatMap((columnNodeDef) => new ColumnNodeDef(this, columnNodeDef.nodeDef).namesFull)
   }
 
+  get columnParentUuidName() {
+    return this.viewDataParent?.columnUuidName
+  }
+
   get tableData() {
     return this._tableData
   }

core/i18n/resources/en/common.js

@@ -392,8 +392,8 @@ Thank you and enjoy **$t(common.appNameFull)**!`,
       activityLog: {
         title: 'Activity log',
       },
-      exportWithData: 'Export with data',
-      exportWithDataNoActivityLog: 'Export with data (NO Activity Log)',
+      exportWithData: 'Export + data (Backup)',
+      exportWithDataNoActivityLog: 'Export + data (NO Activity Log)',
       surveyPropUpdate: {
         main: `<title>Welcome to Arena</title>
   
@@ -560,6 +560,8 @@ $t(common.cantUndoWarning)`,
       includeAnalysis: 'Include result variables',
       includeDataFromAllCycles: 'Include data from all cycles',
       includeFiles: 'Include files',
+      includeFileAttributeDefs: 'Include file attribute columns',
+      includeInternalUuids: 'Include internal UUIDs',
       recordsModifiedAfter: 'Records modified after',
     },
     optionsInfo: {
@@ -1725,7 +1727,7 @@ Levels will be renamed into level_1, level_2... level_N and an extra 'area' prop
       keyDuplicate: 'Duplicate record key',
       entityKeyDuplicate: 'Duplicate key',
       entityKeyValueNotSpecified: 'Key value for "{{keyDefName}}" not specified',
-      missingAncestorForEntity: 'Cannot find ancestor "{{ancestorName}}" for entity "{{entityName}}"',
+      missingAncestorForEntity: 'Cannot find "{{ancestorName}}" with these keys: {{keyValues}}',
       oneOrMoreInvalidValues: 'One or more values are invalid',
       uniqueAttributeDuplicate: 'Duplicate value',
       valueInvalid: 'Invalid value',

core/record/_record/recordNodesUpdater.js

@@ -19,6 +19,19 @@ const { updateNodesDependents } = CoreRecordNodesUpdater
 
 import { afterNodesUpdate } from './recordNodesUpdaterCommon'
 
+const getKeyValuePairs = ({ survey, entityDef, valuesByDefUuid }) => {
+  const keyDefs = Survey.getNodeDefKeys(entityDef)(survey)
+  return keyDefs
+    .map((keyDef) => {
+      const keyDefUuid = NodeDef.getUuid(keyDef)
+      const keyDefName = NodeDef.getName(keyDef)
+      const value = valuesByDefUuid[keyDefUuid]
+      const keyValue = NodeValueFormatter.format({ survey, nodeDef: keyDef, value })
+      return `${keyDefName}=${keyValue}`
+    })
+    .join(',')
+}
+
 const _valueAdapterByType = {
   [NodeDef.nodeDefType.code]: ({ survey, record, parentNode, attributeDef, value }) => {
     if (value[Node.valuePropsCode.itemUuid]) {
@@ -126,16 +139,7 @@ const _getOrCreateEntityByKeys =
     const entityDef = Survey.getNodeDefByUuid(entityDefUuid)(survey)
 
     if (!insertMissingNodes) {
-      const keyDefs = Survey.getNodeDefKeys(entityDef)(survey)
-      const keyValuePairs = keyDefs
-        .map((keyDef) => {
-          const keyDefUuid = NodeDef.getUuid(keyDef)
-          const keyDefName = NodeDef.getName(keyDef)
-          const value = valuesByDefUuid[keyDefUuid]
-          const keyValue = NodeValueFormatter.format({ survey, nodeDef: keyDef, value })
-          return `${keyDefName}=${keyValue}`
-        })
-        .join(',')
+      const keyValuePairs = getKeyValuePairs({ survey, entityDef, valuesByDefUuid })
       throw new SystemError('appErrors:record.entityNotFound', {
         entityName: NodeDef.getName(entityDef),
         keyValues: keyValuePairs,
@@ -153,9 +157,11 @@ const _getOrCreateEntityByKeys =
         })(record)
 
     if (!entityParent) {
+      const keyValuePairs = getKeyValuePairs({ survey, entityDef: entityParentDef, valuesByDefUuid })
       throw new SystemError('validationErrors.record.missingAncestorForEntity', {
         entityName: NodeDef.getName(entityDef),
         ancestorName: NodeDef.getName(entityParentDef),
+        keyValues: keyValuePairs,
       })
     }
     const { entity: entityInserted, updateResult } = _addEntityAndKeyValues({

server/modules/dataExport/api/dataExportApi.js

@@ -16,20 +16,7 @@ export const init = (app) => {
     AuthMiddleware.requireRecordsExportPermission,
     async (req, res, next) => {
       try {
-        const {
-          surveyId,
-          cycle,
-          recordUuids,
-          search,
-          includeCategories,
-          includeCategoryItemsLabels,
-          expandCategoryItems,
-          includeAncestorAttributes,
-          includeAnalysis,
-          includeDataFromAllCycles,
-          includeFiles,
-          recordsModifiedAfter,
-        } = Request.getParams(req)
+        const { surveyId, cycle, recordUuids, search, options } = Request.getParams(req)
 
         const user = Request.getUser(req)
 
@@ -39,14 +26,7 @@ export const init = (app) => {
           cycle,
           recordUuids,
           search,
-          includeCategories,
-          includeCategoryItemsLabels,
-          expandCategoryItems,
-          includeAncestorAttributes,
-          includeAnalysis,
-          includeDataFromAllCycles,
-          includeFiles,
-          recordsModifiedAfter,
+          options,
         })
         res.json({ job: JobUtils.jobToJSON(job) })
       } catch (error) {

server/modules/dataExport/service/DataExportJob/DataExportJob.js

@@ -45,7 +45,8 @@ export default class DataExportJob extends Job {
   }
 
   async _fetchSurvey() {
-    const { surveyId, cycle, includeAnalysis, includeDataFromAllCycles, expandCategoryItems } = this.context
+    const { surveyId, cycle, options } = this.context
+    const { includeAnalysis, includeDataFromAllCycles, expandCategoryItems } = options
     const cycleToFetch = includeDataFromAllCycles ? undefined : cycle
 
     return expandCategoryItems

server/modules/dataExport/service/DataExportJob/jobs/CSVDataExtractionJob.js

@@ -8,20 +8,7 @@ export default class CSVDataExtractionJob extends Job {
   }
 
   async execute() {
-    const {
-      survey,
-      cycle,
-      recordUuids,
-      search,
-      includeCategoryItemsLabels,
-      expandCategoryItems,
-      includeAncestorAttributes,
-      includeAnalysis,
-      includeDataFromAllCycles,
-      includeFiles,
-      recordsModifiedAfter,
-      outputDir,
-    } = this.context
+    const { survey, cycle, recordUuids, search, options, outputDir } = this.context
 
     await SurveyRdbService.fetchEntitiesDataToCsvFiles(
       {
@@ -30,13 +17,7 @@ export default class CSVDataExtractionJob extends Job {
         cycle,
         recordUuids,
         search,
-        includeCategoryItemsLabels,
-        expandCategoryItems,
-        includeAncestorAttributes,
-        includeAnalysis,
-        includeDataFromAllCycles,
-        includeFiles,
-        recordsModifiedAfter,
+        options,
         outputDir,
         callback: ({ total }) => {
           this.total = total

server/modules/dataExport/service/dataExportService.js

@@ -1,35 +1,14 @@
 import * as JobManager from '@server/job/jobManager'
 import DataExportJob from '@server/modules/dataExport/service/DataExportJob'
 
-export const startCsvDataExportJob = ({
-  user,
-  surveyId,
-  cycle,
-  recordUuids,
-  search,
-  includeCategories,
-  includeCategoryItemsLabels,
-  expandCategoryItems,
-  includeAncestorAttributes,
-  includeAnalysis,
-  includeDataFromAllCycles,
-  includeFiles,
-  recordsModifiedAfter,
-}) => {
+export const startCsvDataExportJob = ({ user, surveyId, cycle, recordUuids, search, options }) => {
   const job = new DataExportJob({
     user,
     surveyId,
     cycle,
     recordUuids,
     search,
-    includeCategories,
-    includeCategoryItemsLabels,
-    expandCategoryItems,
-    includeAncestorAttributes,
-    includeAnalysis,
-    includeDataFromAllCycles,
-    includeFiles,
-    recordsModifiedAfter,
+    options,
   })
 
   JobManager.executeJobThread(job)

server/modules/surveyRdb/manager/surveyRdbCsvExport.js

@@ -3,6 +3,7 @@ import * as Survey from '@core/survey/survey'
 import * as NodeDef from '@core/survey/nodeDef'
 import * as CategoryItem from '@core/survey/categoryItem'
 
+import { ArrayUtils } from '@core/arrayUtils'
 import { Query } from '@common/model/query'
 import { CsvDataExportModel } from '@common/model/csvExport'
 import { ColumnNodeDef, ViewDataNodeDef } from '@common/model/db'
@@ -66,29 +67,40 @@ const getCsvExportFields = ({
   addCycle = false,
   includeCategoryItemsLabels = true,
   expandCategoryItems = false,
+  includeInternalUuids = false,
 }) => {
   const entityDef = Survey.getNodeDefByUuid(Query.getEntityDefUuid(query))(survey)
   const viewDataNodeDef = new ViewDataNodeDef(survey, entityDef)
 
   // Consider only user selected fields (from column node defs)
   const nodeDefUuidCols = Query.getAttributeDefUuids(query)
   const nodeDefCols = Survey.getNodeDefsByUuids(nodeDefUuidCols)(survey)
-  const fields = nodeDefCols.flatMap((nodeDefCol) => {
-    const columnNodeDef = new ColumnNodeDef(viewDataNodeDef, nodeDefCol)
-    const fieldsExtractor = fieldsExtractorByNodeDefType[NodeDef.getType(nodeDefCol)]
-    if (fieldsExtractor) {
-      return fieldsExtractor({
-        survey,
-        columnNodeDef,
-        includeCategoryItemsLabels,
-        expandCategoryItems,
-      })
-    } else {
-      return columnNodeDef.names
-    }
-  })
-  // Cycle is 0-based
-  return [...(addCycle ? [DataTable.columnNameRecordCycle] : []), ...fields]
+  const fields = []
+  if (includeInternalUuids) {
+    ArrayUtils.addIfNotEmpty(viewDataNodeDef.columnParentUuidName)(fields)
+    fields.push(viewDataNodeDef.columnUuidName)
+  }
+  if (addCycle) {
+    // Cycle is 0-based
+    fields.push(DataTable.columnNameRecordCycle)
+  }
+  fields.push(
+    ...nodeDefCols.flatMap((nodeDefCol) => {
+      const columnNodeDef = new ColumnNodeDef(viewDataNodeDef, nodeDefCol)
+      const fieldsExtractor = fieldsExtractorByNodeDefType[NodeDef.getType(nodeDefCol)]
+      if (fieldsExtractor) {
+        return fieldsExtractor({
+          survey,
+          columnNodeDef,
+          includeCategoryItemsLabels,
+          expandCategoryItems,
+        })
+      } else {
+        return columnNodeDef.names
+      }
+    })
+  )
+  return fields
 }
 
 const getCsvExportFieldsAgg = ({ survey, query }) => {

server/modules/surveyRdb/manager/surveyRdbManager.js

@@ -78,6 +78,7 @@ export const fetchViewData = async (params, client = db) => {
     addCycle = false,
     includeCategoryItemsLabels = true,
     expandCategoryItems = false,
+    includeInternalUuids = false,
     nullsToEmpty = false,
   } = params
 
@@ -107,6 +108,7 @@ export const fetchViewData = async (params, client = db) => {
           addCycle,
           includeCategoryItemsLabels,
           expandCategoryItems,
+          includeInternalUuids,
         })
     await db.stream(result, (dbStream) => {
       const csvTransform = CSVWriter.transformJsonToCsv({
@@ -167,8 +169,8 @@ export const fetchViewDataAgg = async (params) => {
   return result
 }
 
-const _determineRecordUuidsFilter = async ({ survey, cycle, recordsModifiedAfter, recordUuidsParam, search }) => {
-  if (recordUuidsParam) return recordUuidsParam
+const _determineRecordUuidsFilter = async ({ survey, cycle, recordsModifiedAfter, recordUuids, search }) => {
+  if (recordUuids) return recordUuids
 
   if (Objects.isEmpty(search) && !recordsModifiedAfter) return null
 
@@ -194,20 +196,26 @@ export const fetchEntitiesDataToCsvFiles = async (
   {
     survey,
     cycle,
-    includeCategoryItemsLabels,
-    expandCategoryItems,
-    includeAncestorAttributes,
-    includeAnalysis,
-    includeFiles,
+    search = null,
     recordOwnerUuid = null,
     recordUuids: recordUuidsParam = null,
-    recordsModifiedAfter,
-    search = null,
+    options,
     outputDir,
     callback,
   },
   client = db
 ) => {
+  const {
+    includeCategoryItemsLabels,
+    expandCategoryItems,
+    includeAncestorAttributes,
+    includeAnalysis,
+    includeFileAttributeDefs,
+    includeFiles,
+    includeInternalUuids,
+    recordsModifiedAfter,
+  } = options
+
   const addCycle = Survey.getCycleKeys(survey).length > 1
 
   const nodeDefs = Survey.findDescendants({
@@ -218,7 +226,7 @@ export const fetchEntitiesDataToCsvFiles = async (
     survey,
     cycle,
     recordsModifiedAfter,
-    recordUuidsParam,
+    recordUuids: recordUuidsParam,
     search,
   })
 
@@ -249,7 +257,7 @@ export const fetchEntitiesDataToCsvFiles = async (
         ? getChildAttributes(nodeDefContext)
         : // Multiple attribute
           [nodeDefContext]
-    ).filter((childDef) => includeFiles || !NodeDef.isFile(childDef))
+    ).filter((childDef) => includeFileAttributeDefs || includeFiles || !NodeDef.isFile(childDef))
 
     const ancestorDefs = []
     if (includeAncestorAttributes) {
@@ -281,6 +289,7 @@ export const fetchEntitiesDataToCsvFiles = async (
         addCycle,
         includeCategoryItemsLabels,
         expandCategoryItems,
+        includeInternalUuids,
       },
       client
     )