CSV data export: added option to export UUIDs

openforis · Oct 4, 2024 · 3c33285 · 3c33285
1 parent aa1871c
commit 3c33285
Show file tree

Hide file tree

Showing 12 changed files with 94 additions and 124 deletions.
diff --git a/common/model/db/views/dataNodeDef/view.js b/common/model/db/views/dataNodeDef/view.js
@@ -48,6 +48,10 @@ export default class ViewDataNodeDef extends TableDataNodeDef {
     return this.columnNodeDefUuid.nameFull
   }
 
+  get columnUuidName() {
+    return this.columnNodeDefUuid.name
+  }
+
   get columnIdName() {
     const { nodeDef } = this.tableData
     return `_${NodeDef.getName(nodeDef)}_${TableDataNodeDef.columnSet.id}`
@@ -118,6 +122,10 @@ export default class ViewDataNodeDef extends TableDataNodeDef {
     return this.columnNodeDefs.flatMap((columnNodeDef) => new ColumnNodeDef(this, columnNodeDef.nodeDef).namesFull)
   }
 
+  get columnParentUuidName() {
+    return this.viewDataParent?.columnUuidName
+  }
+
   get tableData() {
     return this._tableData
   }

diff --git a/core/i18n/resources/en/common.js b/core/i18n/resources/en/common.js
@@ -555,6 +555,7 @@ $t(common.cantUndoWarning)`,
       includeAnalysis: 'Include result variables',
       includeDataFromAllCycles: 'Include data from all cycles',
       includeFiles: 'Include files',
+      includeInternalUuids: 'Include internal UUIDs',
       recordsModifiedAfter: 'Records modified after',
     },
     optionsInfo: {

diff --git a/server/modules/dataExport/api/dataExportApi.js b/server/modules/dataExport/api/dataExportApi.js
@@ -16,20 +16,7 @@ export const init = (app) => {
     AuthMiddleware.requireRecordsExportPermission,
     async (req, res, next) => {
       try {
-        const {
-          surveyId,
-          cycle,
-          recordUuids,
-          search,
-          includeCategories,
-          includeCategoryItemsLabels,
-          expandCategoryItems,
-          includeAncestorAttributes,
-          includeAnalysis,
-          includeDataFromAllCycles,
-          includeFiles,
-          recordsModifiedAfter,
-        } = Request.getParams(req)
+        const { surveyId, cycle, recordUuids, search, options } = Request.getParams(req)
 
         const user = Request.getUser(req)
 
@@ -39,14 +26,7 @@ export const init = (app) => {
           cycle,
           recordUuids,
           search,
-          includeCategories,
-          includeCategoryItemsLabels,
-          expandCategoryItems,
-          includeAncestorAttributes,
-          includeAnalysis,
-          includeDataFromAllCycles,
-          includeFiles,
-          recordsModifiedAfter,
+          options,
         })
         res.json({ job: JobUtils.jobToJSON(job) })
       } catch (error) {

diff --git a/server/modules/dataExport/service/DataExportJob/DataExportJob.js b/server/modules/dataExport/service/DataExportJob/DataExportJob.js
@@ -45,7 +45,8 @@ export default class DataExportJob extends Job {
   }
 
   async _fetchSurvey() {
-    const { surveyId, cycle, includeAnalysis, includeDataFromAllCycles, expandCategoryItems } = this.context
+    const { surveyId, cycle, options } = this.context
+    const { includeAnalysis, includeDataFromAllCycles, expandCategoryItems } = options
     const cycleToFetch = includeDataFromAllCycles ? undefined : cycle
 
     return expandCategoryItems

diff --git a/server/modules/dataExport/service/DataExportJob/jobs/CSVDataExtractionJob.js b/server/modules/dataExport/service/DataExportJob/jobs/CSVDataExtractionJob.js
@@ -8,20 +8,7 @@ export default class CSVDataExtractionJob extends Job {
   }
 
   async execute() {
-    const {
-      survey,
-      cycle,
-      recordUuids,
-      search,
-      includeCategoryItemsLabels,
-      expandCategoryItems,
-      includeAncestorAttributes,
-      includeAnalysis,
-      includeDataFromAllCycles,
-      includeFiles,
-      recordsModifiedAfter,
-      outputDir,
-    } = this.context
+    const { survey, cycle, recordUuids, search, options, outputDir } = this.context
 
     await SurveyRdbService.fetchEntitiesDataToCsvFiles(
       {
@@ -30,13 +17,7 @@ export default class CSVDataExtractionJob extends Job {
         cycle,
         recordUuids,
         search,
-        includeCategoryItemsLabels,
-        expandCategoryItems,
-        includeAncestorAttributes,
-        includeAnalysis,
-        includeDataFromAllCycles,
-        includeFiles,
-        recordsModifiedAfter,
+        options,
         outputDir,
         callback: ({ total }) => {
           this.total = total

diff --git a/server/modules/dataExport/service/dataExportService.js b/server/modules/dataExport/service/dataExportService.js
@@ -1,35 +1,14 @@
 import * as JobManager from '@server/job/jobManager'
 import DataExportJob from '@server/modules/dataExport/service/DataExportJob'
 
-export const startCsvDataExportJob = ({
-  user,
-  surveyId,
-  cycle,
-  recordUuids,
-  search,
-  includeCategories,
-  includeCategoryItemsLabels,
-  expandCategoryItems,
-  includeAncestorAttributes,
-  includeAnalysis,
-  includeDataFromAllCycles,
-  includeFiles,
-  recordsModifiedAfter,
-}) => {
+export const startCsvDataExportJob = ({ user, surveyId, cycle, recordUuids, search, options }) => {
   const job = new DataExportJob({
     user,
     surveyId,
     cycle,
     recordUuids,
     search,
-    includeCategories,
-    includeCategoryItemsLabels,
-    expandCategoryItems,
-    includeAncestorAttributes,
-    includeAnalysis,
-    includeDataFromAllCycles,
-    includeFiles,
-    recordsModifiedAfter,
+    options,
   })
 
   JobManager.executeJobThread(job)

diff --git a/server/modules/surveyRdb/manager/surveyRdbCsvExport.js b/server/modules/surveyRdb/manager/surveyRdbCsvExport.js
@@ -3,6 +3,7 @@ import * as Survey from '@core/survey/survey'
 import * as NodeDef from '@core/survey/nodeDef'
 import * as CategoryItem from '@core/survey/categoryItem'
 
+import { ArrayUtils } from '@core/arrayUtils'
 import { Query } from '@common/model/query'
 import { CsvDataExportModel } from '@common/model/csvExport'
 import { ColumnNodeDef, ViewDataNodeDef } from '@common/model/db'
@@ -66,29 +67,40 @@ const getCsvExportFields = ({
   addCycle = false,
   includeCategoryItemsLabels = true,
   expandCategoryItems = false,
+  includeInternalUuids = false,
 }) => {
   const entityDef = Survey.getNodeDefByUuid(Query.getEntityDefUuid(query))(survey)
   const viewDataNodeDef = new ViewDataNodeDef(survey, entityDef)
 
   // Consider only user selected fields (from column node defs)
   const nodeDefUuidCols = Query.getAttributeDefUuids(query)
   const nodeDefCols = Survey.getNodeDefsByUuids(nodeDefUuidCols)(survey)
-  const fields = nodeDefCols.flatMap((nodeDefCol) => {
-    const columnNodeDef = new ColumnNodeDef(viewDataNodeDef, nodeDefCol)
-    const fieldsExtractor = fieldsExtractorByNodeDefType[NodeDef.getType(nodeDefCol)]
-    if (fieldsExtractor) {
-      return fieldsExtractor({
-        survey,
-        columnNodeDef,
-        includeCategoryItemsLabels,
-        expandCategoryItems,
-      })
-    } else {
-      return columnNodeDef.names
-    }
-  })
-  // Cycle is 0-based
-  return [...(addCycle ? [DataTable.columnNameRecordCycle] : []), ...fields]
+  const fields = []
+  if (includeInternalUuids) {
+    ArrayUtils.addIfNotEmpty(viewDataNodeDef.columnParentUuidName)(fields)
+    fields.push(viewDataNodeDef.columnUuidName)
+  }
+  if (addCycle) {
+    // Cycle is 0-based
+    fields.push(DataTable.columnNameRecordCycle)
+  }
+  fields.push(
+    ...nodeDefCols.flatMap((nodeDefCol) => {
+      const columnNodeDef = new ColumnNodeDef(viewDataNodeDef, nodeDefCol)
+      const fieldsExtractor = fieldsExtractorByNodeDefType[NodeDef.getType(nodeDefCol)]
+      if (fieldsExtractor) {
+        return fieldsExtractor({
+          survey,
+          columnNodeDef,
+          includeCategoryItemsLabels,
+          expandCategoryItems,
+        })
+      } else {
+        return columnNodeDef.names
+      }
+    })
+  )
+  return fields
 }
 
 const getCsvExportFieldsAgg = ({ survey, query }) => {

diff --git a/server/modules/surveyRdb/manager/surveyRdbManager.js b/server/modules/surveyRdb/manager/surveyRdbManager.js
@@ -78,6 +78,7 @@ export const fetchViewData = async (params, client = db) => {
     addCycle = false,
     includeCategoryItemsLabels = true,
     expandCategoryItems = false,
+    includeInternalUuids = false,
     nullsToEmpty = false,
   } = params
 
@@ -107,6 +108,7 @@ export const fetchViewData = async (params, client = db) => {
           addCycle,
           includeCategoryItemsLabels,
           expandCategoryItems,
+          includeInternalUuids,
         })
     await db.stream(result, (dbStream) => {
       const csvTransform = CSVWriter.transformJsonToCsv({
@@ -167,8 +169,8 @@ export const fetchViewDataAgg = async (params) => {
   return result
 }
 
-const _determineRecordUuidsFilter = async ({ survey, cycle, recordsModifiedAfter, recordUuidsParam, search }) => {
-  if (recordUuidsParam) return recordUuidsParam
+const _determineRecordUuidsFilter = async ({ survey, cycle, recordsModifiedAfter, recordUuids, search }) => {
+  if (recordUuids) return recordUuids
 
   if (Objects.isEmpty(search) && !recordsModifiedAfter) return null
 
@@ -194,20 +196,25 @@ export const fetchEntitiesDataToCsvFiles = async (
   {
     survey,
     cycle,
-    includeCategoryItemsLabels,
-    expandCategoryItems,
-    includeAncestorAttributes,
-    includeAnalysis,
-    includeFiles,
+    search = null,
     recordOwnerUuid = null,
     recordUuids: recordUuidsParam = null,
-    recordsModifiedAfter,
-    search = null,
+    options,
     outputDir,
     callback,
   },
   client = db
 ) => {
+  const {
+    includeCategoryItemsLabels,
+    expandCategoryItems,
+    includeAncestorAttributes,
+    includeAnalysis,
+    includeFiles,
+    includeInternalUuids,
+    recordsModifiedAfter,
+  } = options
+
   const addCycle = Survey.getCycleKeys(survey).length > 1
 
   const nodeDefs = Survey.findDescendants({
@@ -218,7 +225,7 @@ export const fetchEntitiesDataToCsvFiles = async (
     survey,
     cycle,
     recordsModifiedAfter,
-    recordUuidsParam,
+    recordUuids: recordUuidsParam,
     search,
   })
 
@@ -281,6 +288,7 @@ export const fetchEntitiesDataToCsvFiles = async (
         addCycle,
         includeCategoryItemsLabels,
         expandCategoryItems,
+        includeInternalUuids,
       },
       client
     )

diff --git a/server/modules/surveyRdb/service/surveyRdbService.js b/server/modules/surveyRdb/service/surveyRdbService.js
@@ -148,33 +148,23 @@ export const fetchEntitiesDataToCsvFiles = async ({
   cycle: cycleParam,
   recordUuids,
   search,
-  includeCategoryItemsLabels,
-  expandCategoryItems,
-  includeAncestorAttributes,
-  includeAnalysis,
-  includeDataFromAllCycles,
-  includeFiles,
-  recordsModifiedAfter,
+  options,
   outputDir,
   callback,
 }) => {
   const recordOwnerUuid = _getRecordOwnerUuidForQuery({ user, survey })
 
+  const { includeDataFromAllCycles } = options
   const cycle = includeDataFromAllCycles ? null : cycleParam
 
   return SurveyRdbManager.fetchEntitiesDataToCsvFiles({
     survey,
     cycle,
     outputDir,
     recordUuids,
-    search,
-    includeCategoryItemsLabels,
-    expandCategoryItems,
-    includeAncestorAttributes,
-    includeAnalysis,
-    includeFiles,
     recordOwnerUuid,
-    recordsModifiedAfter,
+    search,
+    options,
     callback,
   })
 }

diff --git a/webapp/service/api/data/index.js b/webapp/service/api/data/index.js
@@ -102,7 +102,7 @@ export const startExportDataToCSVJob = async ({ surveyId, cycle, recordUuids, se
     cycle,
     recordUuids,
     search,
-    ...options,
+    options,
   })
   const { job } = data
   return job

diff --git a/webapp/views/App/views/Data/DataExport/DataExportOptionsPanel.js b/webapp/views/App/views/Data/DataExport/DataExportOptionsPanel.js
@@ -9,7 +9,7 @@ import { FormItem } from '@webapp/components/form/Input'
 import { DateInput } from '@webapp/components/form/DateTimeInput'
 
 import { useSurveyCycleKeys } from '@webapp/store/survey'
-import { useAuthCanUseAnalysis } from '@webapp/store/user'
+import { useAuthCanUseAnalysis, useUserIsSystemAdmin } from '@webapp/store/user'
 import { useI18n } from '@webapp/store/system'
 
 import { dataExportOptions } from './dataExportOptions'
@@ -22,25 +22,33 @@ export const DataExportOptionsPanel = (props) => {
   const { availableOptions: availableOptionsProp, onOptionChange, selectedOptionsByKey } = props
 
   const i18n = useI18n()
+  const isSystemAdmin = useUserIsSystemAdmin()
   const canAnalyzeRecords = useAuthCanUseAnalysis()
   const cycles = useSurveyCycleKeys()
   const hasMultipleCycles = cycles.length > 1
 
-  const availableOptions = useMemo(
-    () =>
-      Objects.isEmpty(availableOptionsProp)
-        ? [
-            dataExportOptions.includeCategoryItemsLabels,
-            dataExportOptions.expandCategoryItems,
-            dataExportOptions.includeCategories,
-            dataExportOptions.includeAncestorAttributes,
-            ...(canAnalyzeRecords ? [dataExportOptions.includeAnalysis] : []),
-            ...(hasMultipleCycles ? [dataExportOptions.includeDataFromAllCycles] : []),
-            dataExportOptions.includeFiles,
-          ]
-        : availableOptionsProp,
-    [availableOptionsProp, canAnalyzeRecords, hasMultipleCycles]
-  )
+  const availableOptions = useMemo(() => {
+    if (Objects.isNotEmpty(availableOptionsProp)) {
+      return availableOptionsProp
+    }
+    const options = [
+      dataExportOptions.includeCategoryItemsLabels,
+      dataExportOptions.expandCategoryItems,
+      dataExportOptions.includeCategories,
+      dataExportOptions.includeAncestorAttributes,
+      dataExportOptions.includeFiles,
+    ]
+    if (canAnalyzeRecords) {
+      options.push(dataExportOptions.includeAnalysis)
+    }
+    if (hasMultipleCycles) {
+      options.push(dataExportOptions.includeDataFromAllCycles)
+    }
+    if (isSystemAdmin) {
+      options.push(dataExportOptions.includeInternalUuids)
+    }
+    return options
+  }, [availableOptionsProp, canAnalyzeRecords, hasMultipleCycles, isSystemAdmin])
 
   return (
     <ExpansionPanel className="options" buttonLabel="dataExportView.options.header">