CSV data export: expand category items option (#3194)

* data export: expand category items (WIP) * data export: expand category items (WIP) * fixed data export * added info message to option * code cleanup * limit expandable category items --------- Co-authored-by: Stefano Ricci <[email protected]> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
openforis · Dec 15, 2023 · 404cd80 · 404cd80
1 parent a0f22f5
commit 404cd80
Show file tree

Hide file tree

Showing 20 changed files with 318 additions and 108 deletions.
diff --git a/common/model/csvExport/CsvDataExportModel.js b/common/model/csvExport/CsvDataExportModel.js
@@ -8,25 +8,39 @@ const columnDataType = {
   text: 'text',
 }
 
+const getExpandedCategoryItemColumnHeader = ({ nodeDef, code }) => `${NodeDef.getName(nodeDef)}__${code}`
+
 const getMainColumn = ({ nodeDef, dataType }) => ({ header: NodeDef.getName(nodeDef), nodeDef, dataType })
 
 const columnsByNodeDefType = {
   [NodeDef.nodeDefType.boolean]: ({ nodeDef }) => [getMainColumn({ nodeDef, dataType: columnDataType.boolean })],
-  [NodeDef.nodeDefType.code]: ({ nodeDef, includeCategoryItemsLabels }) => {
+  [NodeDef.nodeDefType.code]: ({ survey, nodeDef, includeCategoryItemsLabels, expandCategoryItems }) => {
     const nodeDefName = NodeDef.getName(nodeDef)
-    return [
+    const result = [
       { header: nodeDefName, nodeDef, dataType: columnDataType.text, valueProp: Node.valuePropsCode.code },
-      ...(includeCategoryItemsLabels
-        ? [
-            {
-              header: `${nodeDefName}_label`,
-              nodeDef,
-              dataType: columnDataType.text,
-              valueProp: Node.valuePropsCode.label,
-            },
-          ]
-        : []),
     ]
+    if (includeCategoryItemsLabels) {
+      result.push({
+        header: `${nodeDefName}_label`,
+        nodeDef,
+        dataType: columnDataType.text,
+        valueProp: Node.valuePropsCode.label,
+      })
+    }
+    if (expandCategoryItems) {
+      const categoryUuid = NodeDef.getCategoryUuid(nodeDef)
+      const levelIndex = Survey.getNodeDefCategoryLevelIndex(nodeDef)(survey)
+      const items = Survey.getCategoryItemsInLevel({ categoryUuid, levelIndex })(survey)
+      items.forEach((item) => {
+        result.push({
+          header: getExpandedCategoryItemColumnHeader({ nodeDef, code: item.getCode(item) }),
+          nodeDef,
+          dataType: columnDataType.boolean,
+          valueProp: Node.valuePropsCode.label,
+        })
+      })
+    }
+    return result
   },
   [NodeDef.nodeDefType.coordinate]: ({ nodeDef }) => {
     const nodeDefName = NodeDef.getName(nodeDef)
@@ -91,6 +105,7 @@ const DEFAULT_OPTIONS = {
   includeAnalysis: true,
   includeAncestorAttributes: false,
   includeCategoryItemsLabels: true,
+  expandCategoryItems: false,
   includeReadOnlyAttributes: true,
   includeTaxonScientificName: true,
   includeFiles: true,
@@ -128,13 +143,14 @@ export class CsvDataExportModel {
   }
 
   _createColumnsFromAttributeDefs(attributeDefs) {
+    const { survey } = this
     const { includeCategoryItemsLabels, includeTaxonScientificName } = this.options
 
     return attributeDefs.reduce((acc, nodeDef) => {
       const columnsGetter = columnsByNodeDefType[NodeDef.getType(nodeDef)]
 
       const columnsPerAttribute = columnsGetter
-        ? columnsGetter({ nodeDef, includeCategoryItemsLabels, includeTaxonScientificName })
+        ? columnsGetter({ survey, nodeDef, includeCategoryItemsLabels, includeTaxonScientificName })
         : []
 
       if (NodeDef.isKey(nodeDef)) {
@@ -198,4 +214,6 @@ export class CsvDataExportModel {
   }
 }
 
+CsvDataExportModel.getExpandedCategoryItemColumnHeader = getExpandedCategoryItemColumnHeader
+
 CsvDataExportModel.columnDataType = columnDataType
diff --git a/core/i18n/resources/en.js b/core/i18n/resources/en.js
@@ -555,10 +555,14 @@ $t(common.cantUndoWarning)`,
       header: '$t(common.options)',
       includeCategoryItemsLabels: 'Include category items labels',
       includeCategories: 'Include categories',
+      expandCategoryItems: 'Expand category items',
       includeAnalysis: 'Include result variables',
       includeDataFromAllCycles: 'Include data from all cycles',
       includeFiles: 'Include files',
     },
+    optionsInfo: {
+      expandCategoryItems: 'add one boolean column for every category item',
+    },
     startExport: 'Start export',
   },
 

diff --git a/core/survey/_survey/surveyNodeDefs.js b/core/survey/_survey/surveyNodeDefs.js
@@ -93,7 +93,7 @@ export const getNodeDefDescendantsInSingleEntities =
         ? getNodeDefChildrenSorted({ nodeDef: entityDefCurrent, includeAnalysis, cycle })(survey)
         : getNodeDefChildren(entityDefCurrent, includeAnalysis)(survey)
 
-      descendants.push(...entityDefCurrentChildren.filter(filterFn))
+      descendants.push(...(filterFn ? entityDefCurrentChildren.filter(filterFn) : entityDefCurrentChildren))
 
       // visit nodes inside single entities
       queue.enqueueItems(entityDefCurrentChildren.filter(NodeDef.isSingleEntity))
@@ -116,13 +116,14 @@ export const getNodeDefDescendantsInSingleEntities =
 export const getNodeDefDescendantAttributesInSingleEntities = ({
   nodeDef,
   includeAnalysis = false,
+  includeMultipleAttributes = false,
   sorted = false,
   cycle = null,
 }) =>
   getNodeDefDescendantsInSingleEntities({
     nodeDef,
     includeAnalysis,
-    filterFn: NodeDef.isSingleAttribute,
+    filterFn: includeMultipleAttributes ? NodeDef.isAttribute : NodeDef.isSingleAttribute,
     sorted,
     cycle,
   })

diff --git a/core/survey/_survey/surveyRefDataIndex.js b/core/survey/_survey/surveyRefDataIndex.js
@@ -3,6 +3,7 @@ import * as RecordReader from '@core/record/_record/recordReader'
 import * as Node from '@core/record/node'
 
 import * as CategoryLevel from '@core/survey/categoryLevel'
+import * as CategoryItem from '@core/survey/categoryItem'
 import * as Taxon from '@core/survey/taxon'
 import * as SurveyNodeDefs from './surveyNodeDefs'
 import { Objects, SurveyRefDataFactory, Surveys } from '@openforis/arena-core'
@@ -88,6 +89,26 @@ export const getCategoryItemByHierarchicalCodes =
   (survey) =>
     Surveys.getCategoryItemByCodePaths({ survey, categoryUuid, codePaths: codesPath })
 
+export const getCategoryItemsInLevel =
+  ({ categoryUuid, levelIndex = 0 }) =>
+  (survey) => {
+    const rootItems = Surveys.getCategoryItems({ survey, categoryUuid, parentItemUuid: null })
+    if (levelIndex === 0) return rootItems
+
+    let itemsPrevLevel = [...rootItems]
+    let currentLevelIndex = 1
+    while (currentLevelIndex <= levelIndex) {
+      const itemsInLevel = []
+      itemsPrevLevel.forEach((item) => {
+        const parentItemUuid = CategoryItem.getUuid(item)
+        itemsInLevel.push(...Surveys.getCategoryItems({ survey, categoryUuid, parentItemUuid }))
+      })
+      currentLevelIndex += 1
+      itemsPrevLevel = itemsInLevel
+    }
+    return itemsPrevLevel
+  }
+
 // ==== taxonomy index
 
 export const getTaxonByUuid = (taxonUuid) => (survey) => Surveys.getTaxonByUuid({ survey, taxonUuid })

diff --git a/core/survey/survey.js b/core/survey/survey.js
@@ -4,6 +4,8 @@ import { uuidv4 } from '@core/uuid'
 
 import * as Srs from '@core/geo/srs'
 
+import * as NodeDef from './nodeDef'
+
 import * as SurveySortKeys from './_survey/surveySortKeys'
 
 import * as SurveyInfo from './_survey/surveyInfo'
@@ -267,8 +269,19 @@ export const { getTaxonomiesArray, getTaxonomyByName, getTaxonomyByUuid, assocTa
 
 // ====== Survey Reference data index
 // Category index
-export const { getCategoryItemUuidAndCodeHierarchy, getCategoryItemByUuid, getCategoryItemByHierarchicalCodes } =
-  SurveyRefDataIndex
+export const {
+  getCategoryItemUuidAndCodeHierarchy,
+  getCategoryItemByUuid,
+  getCategoryItemByHierarchicalCodes,
+  getCategoryItemsInLevel,
+} = SurveyRefDataIndex
+
+export const getNodeDefCategoryItems = (nodeDef) => (survey) => {
+  const categoryUuid = NodeDef.getCategoryUuid(nodeDef)
+  const levelIndex = SurveyNodeDefs.getNodeDefCategoryLevelIndex(nodeDef)(survey)
+  return SurveyRefDataIndex.getCategoryItemsInLevel({ categoryUuid, levelIndex })(survey)
+}
+
 // Taxon index
 export const {
   getTaxonByCode,

diff --git a/server/modules/dataExport/api/dataExportApi.js b/server/modules/dataExport/api/dataExportApi.js
@@ -24,6 +24,7 @@ export const init = (app) => {
           search,
           includeCategories,
           includeCategoryItemsLabels,
+          expandCategoryItems,
           includeAnalysis,
           includeDataFromAllCycles,
           includeFiles,
@@ -39,6 +40,7 @@ export const init = (app) => {
           search,
           includeCategories,
           includeCategoryItemsLabels,
+          expandCategoryItems,
           includeAnalysis,
           includeDataFromAllCycles,
           includeFiles,

diff --git a/server/modules/survey/manager/surveyManager.js b/server/modules/survey/manager/surveyManager.js
@@ -256,11 +256,20 @@ export const fetchSurveyAndNodeDefsBySurveyId = async (
 }
 
 export const fetchSurveyAndNodeDefsAndRefDataBySurveyId = async (
-  { surveyId, cycle = null, draft = false, advanced = false, validate = false, includeDeleted = false, backup = false },
+  {
+    surveyId,
+    cycle = null,
+    draft = false,
+    advanced = false,
+    validate = false,
+    includeDeleted = false,
+    includeAnalysis = true,
+    backup = false,
+  },
   client = db
 ) => {
   const survey = await fetchSurveyAndNodeDefsBySurveyId(
-    { surveyId, cycle, draft, advanced, validate, includeDeleted, backup },
+    { surveyId, cycle, draft, advanced, validate, includeDeleted, includeAnalysis, backup },
     client
   )
   const categoryItemsRefData = await CategoryRepository.fetchIndex(surveyId, draft, client)

diff --git a/server/modules/survey/service/export/exportCsvDataJob.js b/server/modules/survey/service/export/exportCsvDataJob.js
@@ -24,8 +24,6 @@ export default class ExportCsvDataJob extends Job {
   async onStart() {
     await super.onStart()
 
-    const { surveyId, cycle, includeAnalysis, includeDataFromAllCycles } = this.context
-
     // exportUuid will be used when dowloading the generated output file
     // the generated zip file will be named `${exportUuid}.zip`
     const exportUuid = this.uuid
@@ -37,11 +35,7 @@ export default class ExportCsvDataJob extends Job {
 
     await FileUtils.mkdir(outputDir)
 
-    const survey = await SurveyService.fetchSurveyAndNodeDefsBySurveyId({
-      surveyId,
-      cycle: includeDataFromAllCycles ? undefined : cycle,
-      includeAnalysis,
-    })
+    const survey = await this._fetchSurvey()
 
     this.setContext({
       exportUuid,
@@ -50,6 +44,23 @@ export default class ExportCsvDataJob extends Job {
     })
   }
 
+  async _fetchSurvey() {
+    const { surveyId, cycle, includeAnalysis, includeDataFromAllCycles, expandCategoryItems } = this.context
+    const cycleToFetch = includeDataFromAllCycles ? undefined : cycle
+
+    return expandCategoryItems
+      ? await SurveyService.fetchSurveyAndNodeDefsAndRefDataBySurveyId({
+          surveyId,
+          cycle: cycleToFetch,
+          includeAnalysis,
+        })
+      : await SurveyService.fetchSurveyAndNodeDefsBySurveyId({
+          surveyId,
+          cycle: cycleToFetch,
+          includeAnalysis,
+        })
+  }
+
   async beforeSuccess() {
     await super.beforeSuccess()
 

diff --git a/server/modules/survey/service/export/jobs/CSVDataExtractionJob.js b/server/modules/survey/service/export/jobs/CSVDataExtractionJob.js
@@ -14,6 +14,7 @@ export default class CSVDataExtractionJob extends Job {
       recordUuids,
       search,
       includeCategoryItemsLabels,
+      expandCategoryItems,
       includeAnalysis,
       includeDataFromAllCycles,
       outputDir,
@@ -27,6 +28,7 @@ export default class CSVDataExtractionJob extends Job {
         recordUuids,
         search,
         includeCategoryItemsLabels,
+        expandCategoryItems,
         includeAnalysis,
         includeDataFromAllCycles,
         outputDir,

diff --git a/server/modules/survey/service/surveyService.js b/server/modules/survey/service/surveyService.js
@@ -57,6 +57,7 @@ export const startExportCsvDataJob = ({
   search,
   includeCategories,
   includeCategoryItemsLabels,
+  expandCategoryItems,
   includeAnalysis,
   includeDataFromAllCycles,
   includeFiles,
@@ -69,6 +70,7 @@ export const startExportCsvDataJob = ({
     search,
     includeCategories,
     includeCategoryItemsLabels,
+    expandCategoryItems,
     includeAnalysis,
     includeDataFromAllCycles,
     includeFiles,