Data export: include ancestor attributes (#3209)

* data export: include ancestor attributes * code cleanup * layout adjustments --------- Co-authored-by: Stefano Ricci <[email protected]>
openforis · Dec 21, 2023 · 53dafc3 · 53dafc3
1 parent 496d75d
commit 53dafc3
Show file tree

Hide file tree

Showing 14 changed files with 75 additions and 39 deletions.
diff --git a/core/i18n/resources/en.js b/core/i18n/resources/en.js
@@ -556,6 +556,7 @@ $t(common.cantUndoWarning)`,
       includeCategoryItemsLabels: 'Include category items labels',
       includeCategories: 'Include categories',
       expandCategoryItems: 'Expand category items',
+      includeAncestorAttributes: 'Include ancestor attributes',
       includeAnalysis: 'Include result variables',
       includeDataFromAllCycles: 'Include data from all cycles',
       includeFiles: 'Include files',

diff --git a/server/modules/dataExport/api/dataExportApi.js b/server/modules/dataExport/api/dataExportApi.js
@@ -25,6 +25,7 @@ export const init = (app) => {
           includeCategories,
           includeCategoryItemsLabels,
           expandCategoryItems,
+          includeAncestorAttributes,
           includeAnalysis,
           includeDataFromAllCycles,
           includeFiles,
@@ -41,6 +42,7 @@ export const init = (app) => {
           includeCategories,
           includeCategoryItemsLabels,
           expandCategoryItems,
+          includeAncestorAttributes,
           includeAnalysis,
           includeDataFromAllCycles,
           includeFiles,

diff --git a/server/modules/survey/service/export/jobs/CSVDataExtractionJob.js b/server/modules/survey/service/export/jobs/CSVDataExtractionJob.js
@@ -15,6 +15,7 @@ export default class CSVDataExtractionJob extends Job {
       search,
       includeCategoryItemsLabels,
       expandCategoryItems,
+      includeAncestorAttributes,
       includeAnalysis,
       includeDataFromAllCycles,
       outputDir,
@@ -29,6 +30,7 @@ export default class CSVDataExtractionJob extends Job {
         search,
         includeCategoryItemsLabels,
         expandCategoryItems,
+        includeAncestorAttributes,
         includeAnalysis,
         includeDataFromAllCycles,
         outputDir,

diff --git a/server/modules/survey/service/surveyService.js b/server/modules/survey/service/surveyService.js
@@ -58,6 +58,7 @@ export const startExportCsvDataJob = ({
   includeCategories,
   includeCategoryItemsLabels,
   expandCategoryItems,
+  includeAncestorAttributes,
   includeAnalysis,
   includeDataFromAllCycles,
   includeFiles,
@@ -71,6 +72,7 @@ export const startExportCsvDataJob = ({
     includeCategories,
     includeCategoryItemsLabels,
     expandCategoryItems,
+    includeAncestorAttributes,
     includeAnalysis,
     includeDataFromAllCycles,
     includeFiles,

diff --git a/server/modules/surveyRdb/manager/surveyRdbManager.js b/server/modules/surveyRdb/manager/surveyRdbManager.js
@@ -159,12 +159,32 @@ export const fetchViewDataAgg = async (params) => {
   return result
 }
 
+const _determineRecordUuidsFilter = async ({ survey, cycle, recordUuidsParam, search }) => {
+  if (recordUuidsParam) return recordUuidsParam
+
+  if (!Objects.isEmpty(search)) {
+    const surveyId = Survey.getId(survey)
+    const nodeDefRoot = Survey.getNodeDefRoot(survey)
+    const nodeDefKeys = Survey.getNodeDefRootKeys(survey)
+    const recordsSummaries = await RecordRepository.fetchRecordsSummaryBySurveyId({
+      surveyId,
+      nodeDefRoot,
+      nodeDefKeys,
+      cycle,
+      search,
+    })
+    return recordsSummaries.length > 0 ? recordsSummaries.map(Record.getUuid) : null
+  }
+  return null
+}
+
 export const fetchEntitiesDataToCsvFiles = async (
   {
     survey,
     cycle,
     includeCategoryItemsLabels,
     expandCategoryItems,
+    includeAncestorAttributes,
     includeAnalysis,
     recordOwnerUuid = null,
     recordUuids: recordUuidsParam = null,
@@ -180,45 +200,45 @@ export const fetchEntitiesDataToCsvFiles = async (
     (nodeDef) => NodeDef.isRoot(nodeDef) || NodeDef.isMultiple(nodeDef)
   )
 
-  let recordUuids = null
-  if (recordUuidsParam) {
-    recordUuids = recordUuidsParam
-  } else if (!Objects.isEmpty(search)) {
-    const surveyId = Survey.getId(survey)
-    const nodeDefRoot = Survey.getNodeDefRoot(survey)
-    const nodeDefKeys = Survey.getNodeDefRootKeys(survey)
-    const recordsSummaries = await RecordRepository.fetchRecordsSummaryBySurveyId({
-      surveyId,
-      nodeDefRoot,
-      nodeDefKeys,
-      cycle,
-      search,
-    })
-    recordUuids = recordsSummaries.length > 0 ? recordsSummaries.map(Record.getUuid) : null
-  }
+  const recordUuids = await _determineRecordUuidsFilter({ survey, cycle, recordUuidsParam, search })
 
   callback?.({ total: nodeDefs.length })
 
+  const getChildAttributes = (nodeDef) =>
+    Survey.getNodeDefDescendantAttributesInSingleEntities({
+      nodeDef,
+      includeAnalysis,
+      includeMultipleAttributes: !!expandCategoryItems,
+      sorted: true,
+      cycle,
+    })(survey)
+
   await PromiseUtils.each(nodeDefs, async (nodeDefContext, idx) => {
     const entityDefUuid = NodeDef.getUuid(nodeDefContext)
     const outputFilePath = FileUtils.join(outputDir, `${NodeDef.getName(nodeDefContext)}.csv`)
     const outputStream = FileUtils.createWriteStream(outputFilePath)
 
     const childDefs = NodeDef.isEntity(nodeDefContext)
-      ? Survey.getNodeDefDescendantAttributesInSingleEntities({
-          nodeDef: nodeDefContext,
-          includeAnalysis,
-          includeMultipleAttributes: !!expandCategoryItems,
-          sorted: true,
-          cycle,
-        })(survey)
-      : [nodeDefContext] // Multiple attribute
-
-    const ancestorKeysDefs = Survey.getNodeDefAncestorsKeyAttributes(nodeDefContext)(survey)
+      ? getChildAttributes(nodeDefContext)
+      : // Multiple attribute
+        [nodeDefContext]
+
+    const ancestorDefs = []
+    if (includeAncestorAttributes) {
+      Survey.visitAncestors(
+        nodeDefContext,
+        (nodeDef) => {
+          ancestorDefs.push(...getChildAttributes(nodeDef))
+        },
+        false
+      )(survey)
+    } else {
+      ancestorDefs.push(...Survey.getNodeDefAncestorsKeyAttributes(nodeDefContext)(survey))
+    }
 
     let query = Query.create({ entityDefUuid })
-    const ancestorKeyDefsUuids = ancestorKeysDefs.concat(childDefs).map(NodeDef.getUuid)
-    query = Query.assocAttributeDefUuids(ancestorKeyDefsUuids)(query)
+    const ancestorAttributeDefUuids = ancestorDefs.concat(childDefs).map(NodeDef.getUuid)
+    query = Query.assocAttributeDefUuids(ancestorAttributeDefUuids)(query)
     query = Query.assocFilterRecordUuids(recordUuids)(query)
 
     callback?.({ step: idx + 1, total: nodeDefs.length, currentEntity: NodeDef.getName(nodeDefContext) })

diff --git a/server/modules/surveyRdb/service/surveyRdbService.js b/server/modules/surveyRdb/service/surveyRdbService.js
@@ -140,6 +140,7 @@ export const fetchEntitiesDataToCsvFiles = async ({
   search,
   includeCategoryItemsLabels,
   expandCategoryItems,
+  includeAncestorAttributes,
   includeAnalysis,
   includeDataFromAllCycles,
   outputDir,
@@ -157,6 +158,7 @@ export const fetchEntitiesDataToCsvFiles = async ({
     search,
     includeCategoryItemsLabels,
     expandCategoryItems,
+    includeAncestorAttributes,
     includeAnalysis,
     recordOwnerUuid,
     callback,

diff --git a/webapp/components/form/checkbox.js b/webapp/components/form/checkbox.js
@@ -41,7 +41,7 @@ const Checkbox = (props) => {
         >
           <span className={classNameIconContainer} />
           <LabelWithTooltip label={i18n.t(label)} />
-          {info && <ButtonIconInfo title={i18n.t(info)} />}
+          {info && <ButtonIconInfo className="info-icon-btn" title={i18n.t(info)} />}
         </button>
       </ValidationTooltip>
     </div>

diff --git a/webapp/components/form/checkbox.scss b/webapp/components/form/checkbox.scss
@@ -1,4 +1,8 @@
 .btn-checkbox {
   display: inline-flex;
   align-items: center;
+
+  .info-icon-btn {
+    padding: 0 0.5rem;
+  }
 }
diff --git a/webapp/views/App/views/Data/Data.js b/webapp/views/App/views/Data/Data.js
@@ -19,7 +19,7 @@ import ValidationReport from './ValidationReport'
 import Records from './Records'
 import Explorer from './Explorer'
 import Charts from './Charts'
-import ExportData from './ExportData'
+import DataExport from './DataExport'
 import DataImport from './DataImport'
 import { MapView } from './MapView'
 
@@ -83,7 +83,7 @@ const Data = () => {
           ...(Authorizer.canExportRecords(user, surveyInfo)
             ? [
                 {
-                  component: ExportData,
+                  component: DataExport,
                   path: dataModules.export.path,
                 },
               ]

diff --git a/...s/App/views/Data/ExportData/ExportData.js → ...s/App/views/Data/DataExport/DataExport.js b/...s/App/views/Data/ExportData/ExportData.js → ...s/App/views/Data/DataExport/DataExport.js
@@ -1,4 +1,4 @@
-import './ExportData.scss'
+import './DataExport.scss'
 
 import React, { useState } from 'react'
 import { useDispatch } from 'react-redux'
@@ -20,6 +20,7 @@ const exportOptions = {
   includeCategoryItemsLabels: 'includeCategoryItemsLabels',
   expandCategoryItems: 'expandCategoryItems',
   includeCategories: 'includeCategories',
+  includeAncestorAttributes: 'includeAncestorAttributes',
   includeAnalysis: 'includeAnalysis',
   includeDataFromAllCycles: 'includeDataFromAllCycles',
   includeFiles: 'includeFiles',
@@ -33,6 +34,7 @@ const defaultOptionsSelection = {
   [exportOptions.includeCategoryItemsLabels]: true,
   [exportOptions.expandCategoryItems]: false,
   [exportOptions.includeCategories]: false,
+  [exportOptions.includeAncestorAttributes]: false,
   [exportOptions.includeAnalysis]: false,
   [exportOptions.includeDataFromAllCycles]: false,
   [exportOptions.includeFiles]: false,
@@ -44,7 +46,7 @@ const sources = {
   selectedRecords: 'selectedRecords',
 }
 
-const ExportData = (props) => {
+const DataExport = (props) => {
   const { recordUuids, search, sourceSelectionAvailable } = props
 
   const dispatch = useDispatch()
@@ -104,6 +106,7 @@ const ExportData = (props) => {
           exportOptions.includeCategoryItemsLabels,
           exportOptions.expandCategoryItems,
           exportOptions.includeCategories,
+          exportOptions.includeAncestorAttributes,
           ...(canAnalyzeRecords ? [exportOptions.includeAnalysis] : []),
           ...(cycles.length > 1 ? [exportOptions.includeDataFromAllCycles] : []),
           exportOptions.includeFiles,
@@ -128,14 +131,14 @@ const ExportData = (props) => {
   )
 }
 
-ExportData.propTypes = {
+DataExport.propTypes = {
   recordUuids: PropTypes.array,
   search: PropTypes.string,
   sourceSelectionAvailable: PropTypes.bool,
 }
 
-ExportData.defaultProps = {
+DataExport.defaultProps = {
   sourceSelectionAvailable: false,
 }
 
-export default ExportData
+export default DataExport
diff --git a/...App/views/Data/ExportData/ExportData.scss → ...App/views/Data/DataExport/DataExport.scss b/...App/views/Data/ExportData/ExportData.scss → ...App/views/Data/DataExport/DataExport.scss
diff --git a/webapp/views/App/views/Data/DataExport/index.js b/webapp/views/App/views/Data/DataExport/index.js
@@ -0,0 +1 @@
+export { default } from './DataExport'
diff --git a/webapp/views/App/views/Data/ExportData/index.js b/webapp/views/App/views/Data/ExportData/index.js
diff --git a/webapp/views/App/views/Data/Records/HeaderLeft/RecordsDataExportModal.js b/webapp/views/App/views/Data/Records/HeaderLeft/RecordsDataExportModal.js
@@ -4,15 +4,15 @@ import React from 'react'
 import PropTypes from 'prop-types'
 
 import { Modal, ModalBody } from '@webapp/components/modal'
-import ExportData from '../../ExportData'
+import DataExport from '../../DataExport'
 
 export const RecordsDataExportModal = (props) => {
   const { onClose, recordUuids, search } = props
 
   return (
     <Modal className="records-data-export" onClose={onClose} showCloseButton title="dataView.dataExport.title">
       <ModalBody>
-        <ExportData search={search} recordUuids={recordUuids} sourceSelectionAvailable />
+        <DataExport search={search} recordUuids={recordUuids} sourceSelectionAvailable />
       </ModalBody>
     </Modal>
   )