From 53dafc3d04e372f7011c87b4cc72ba6480e6354e Mon Sep 17 00:00:00 2001 From: Stefano Ricci <1219739+SteRiccio@users.noreply.github.com> Date: Thu, 21 Dec 2023 10:02:32 +0100 Subject: [PATCH] Data export: include ancestor attributes (#3209) * data export: include ancestor attributes * code cleanup * layout adjustments --------- Co-authored-by: Stefano Ricci --- core/i18n/resources/en.js | 1 + .../modules/dataExport/api/dataExportApi.js | 2 + .../export/jobs/CSVDataExtractionJob.js | 2 + .../modules/survey/service/surveyService.js | 2 + .../surveyRdb/manager/surveyRdbManager.js | 76 ++++++++++++------- .../surveyRdb/service/surveyRdbService.js | 2 + webapp/components/form/checkbox.js | 2 +- webapp/components/form/checkbox.scss | 4 + webapp/views/App/views/Data/Data.js | 4 +- .../DataExport.js} | 13 ++-- .../DataExport.scss} | 0 .../views/App/views/Data/DataExport/index.js | 1 + .../views/App/views/Data/ExportData/index.js | 1 - .../HeaderLeft/RecordsDataExportModal.js | 4 +- 14 files changed, 75 insertions(+), 39 deletions(-) rename webapp/views/App/views/Data/{ExportData/ExportData.js => DataExport/DataExport.js} (93%) rename webapp/views/App/views/Data/{ExportData/ExportData.scss => DataExport/DataExport.scss} (100%) create mode 100644 webapp/views/App/views/Data/DataExport/index.js delete mode 100644 webapp/views/App/views/Data/ExportData/index.js diff --git a/core/i18n/resources/en.js b/core/i18n/resources/en.js index 7e080c37c1..1eaba053de 100644 --- a/core/i18n/resources/en.js +++ b/core/i18n/resources/en.js @@ -556,6 +556,7 @@ $t(common.cantUndoWarning)`, includeCategoryItemsLabels: 'Include category items labels', includeCategories: 'Include categories', expandCategoryItems: 'Expand category items', + includeAncestorAttributes: 'Include ancestor attributes', includeAnalysis: 'Include result variables', includeDataFromAllCycles: 'Include data from all cycles', includeFiles: 'Include files', diff --git a/server/modules/dataExport/api/dataExportApi.js b/server/modules/dataExport/api/dataExportApi.js index bc69d380c5..f1795c6ef2 100644 --- a/server/modules/dataExport/api/dataExportApi.js +++ b/server/modules/dataExport/api/dataExportApi.js @@ -25,6 +25,7 @@ export const init = (app) => { includeCategories, includeCategoryItemsLabels, expandCategoryItems, + includeAncestorAttributes, includeAnalysis, includeDataFromAllCycles, includeFiles, @@ -41,6 +42,7 @@ export const init = (app) => { includeCategories, includeCategoryItemsLabels, expandCategoryItems, + includeAncestorAttributes, includeAnalysis, includeDataFromAllCycles, includeFiles, diff --git a/server/modules/survey/service/export/jobs/CSVDataExtractionJob.js b/server/modules/survey/service/export/jobs/CSVDataExtractionJob.js index f9dbf924cc..81ec3da36b 100644 --- a/server/modules/survey/service/export/jobs/CSVDataExtractionJob.js +++ b/server/modules/survey/service/export/jobs/CSVDataExtractionJob.js @@ -15,6 +15,7 @@ export default class CSVDataExtractionJob extends Job { search, includeCategoryItemsLabels, expandCategoryItems, + includeAncestorAttributes, includeAnalysis, includeDataFromAllCycles, outputDir, @@ -29,6 +30,7 @@ export default class CSVDataExtractionJob extends Job { search, includeCategoryItemsLabels, expandCategoryItems, + includeAncestorAttributes, includeAnalysis, includeDataFromAllCycles, outputDir, diff --git a/server/modules/survey/service/surveyService.js b/server/modules/survey/service/surveyService.js index 5e5b676fc7..4b370e0ce5 100644 --- a/server/modules/survey/service/surveyService.js +++ b/server/modules/survey/service/surveyService.js @@ -58,6 +58,7 @@ export const startExportCsvDataJob = ({ includeCategories, includeCategoryItemsLabels, expandCategoryItems, + includeAncestorAttributes, includeAnalysis, includeDataFromAllCycles, includeFiles, @@ -71,6 +72,7 @@ export const startExportCsvDataJob = ({ includeCategories, includeCategoryItemsLabels, expandCategoryItems, + includeAncestorAttributes, includeAnalysis, includeDataFromAllCycles, includeFiles, diff --git a/server/modules/surveyRdb/manager/surveyRdbManager.js b/server/modules/surveyRdb/manager/surveyRdbManager.js index bc2355fc48..0ec8a31036 100644 --- a/server/modules/surveyRdb/manager/surveyRdbManager.js +++ b/server/modules/surveyRdb/manager/surveyRdbManager.js @@ -159,12 +159,32 @@ export const fetchViewDataAgg = async (params) => { return result } +const _determineRecordUuidsFilter = async ({ survey, cycle, recordUuidsParam, search }) => { + if (recordUuidsParam) return recordUuidsParam + + if (!Objects.isEmpty(search)) { + const surveyId = Survey.getId(survey) + const nodeDefRoot = Survey.getNodeDefRoot(survey) + const nodeDefKeys = Survey.getNodeDefRootKeys(survey) + const recordsSummaries = await RecordRepository.fetchRecordsSummaryBySurveyId({ + surveyId, + nodeDefRoot, + nodeDefKeys, + cycle, + search, + }) + return recordsSummaries.length > 0 ? recordsSummaries.map(Record.getUuid) : null + } + return null +} + export const fetchEntitiesDataToCsvFiles = async ( { survey, cycle, includeCategoryItemsLabels, expandCategoryItems, + includeAncestorAttributes, includeAnalysis, recordOwnerUuid = null, recordUuids: recordUuidsParam = null, @@ -180,45 +200,45 @@ export const fetchEntitiesDataToCsvFiles = async ( (nodeDef) => NodeDef.isRoot(nodeDef) || NodeDef.isMultiple(nodeDef) ) - let recordUuids = null - if (recordUuidsParam) { - recordUuids = recordUuidsParam - } else if (!Objects.isEmpty(search)) { - const surveyId = Survey.getId(survey) - const nodeDefRoot = Survey.getNodeDefRoot(survey) - const nodeDefKeys = Survey.getNodeDefRootKeys(survey) - const recordsSummaries = await RecordRepository.fetchRecordsSummaryBySurveyId({ - surveyId, - nodeDefRoot, - nodeDefKeys, - cycle, - search, - }) - recordUuids = recordsSummaries.length > 0 ? recordsSummaries.map(Record.getUuid) : null - } + const recordUuids = await _determineRecordUuidsFilter({ survey, cycle, recordUuidsParam, search }) callback?.({ total: nodeDefs.length }) + const getChildAttributes = (nodeDef) => + Survey.getNodeDefDescendantAttributesInSingleEntities({ + nodeDef, + includeAnalysis, + includeMultipleAttributes: !!expandCategoryItems, + sorted: true, + cycle, + })(survey) + await PromiseUtils.each(nodeDefs, async (nodeDefContext, idx) => { const entityDefUuid = NodeDef.getUuid(nodeDefContext) const outputFilePath = FileUtils.join(outputDir, `${NodeDef.getName(nodeDefContext)}.csv`) const outputStream = FileUtils.createWriteStream(outputFilePath) const childDefs = NodeDef.isEntity(nodeDefContext) - ? Survey.getNodeDefDescendantAttributesInSingleEntities({ - nodeDef: nodeDefContext, - includeAnalysis, - includeMultipleAttributes: !!expandCategoryItems, - sorted: true, - cycle, - })(survey) - : [nodeDefContext] // Multiple attribute - - const ancestorKeysDefs = Survey.getNodeDefAncestorsKeyAttributes(nodeDefContext)(survey) + ? getChildAttributes(nodeDefContext) + : // Multiple attribute + [nodeDefContext] + + const ancestorDefs = [] + if (includeAncestorAttributes) { + Survey.visitAncestors( + nodeDefContext, + (nodeDef) => { + ancestorDefs.push(...getChildAttributes(nodeDef)) + }, + false + )(survey) + } else { + ancestorDefs.push(...Survey.getNodeDefAncestorsKeyAttributes(nodeDefContext)(survey)) + } let query = Query.create({ entityDefUuid }) - const ancestorKeyDefsUuids = ancestorKeysDefs.concat(childDefs).map(NodeDef.getUuid) - query = Query.assocAttributeDefUuids(ancestorKeyDefsUuids)(query) + const ancestorAttributeDefUuids = ancestorDefs.concat(childDefs).map(NodeDef.getUuid) + query = Query.assocAttributeDefUuids(ancestorAttributeDefUuids)(query) query = Query.assocFilterRecordUuids(recordUuids)(query) callback?.({ step: idx + 1, total: nodeDefs.length, currentEntity: NodeDef.getName(nodeDefContext) }) diff --git a/server/modules/surveyRdb/service/surveyRdbService.js b/server/modules/surveyRdb/service/surveyRdbService.js index 7128b8c2b9..e9945c09fe 100644 --- a/server/modules/surveyRdb/service/surveyRdbService.js +++ b/server/modules/surveyRdb/service/surveyRdbService.js @@ -140,6 +140,7 @@ export const fetchEntitiesDataToCsvFiles = async ({ search, includeCategoryItemsLabels, expandCategoryItems, + includeAncestorAttributes, includeAnalysis, includeDataFromAllCycles, outputDir, @@ -157,6 +158,7 @@ export const fetchEntitiesDataToCsvFiles = async ({ search, includeCategoryItemsLabels, expandCategoryItems, + includeAncestorAttributes, includeAnalysis, recordOwnerUuid, callback, diff --git a/webapp/components/form/checkbox.js b/webapp/components/form/checkbox.js index c85ea61f15..36c49ac22c 100644 --- a/webapp/components/form/checkbox.js +++ b/webapp/components/form/checkbox.js @@ -41,7 +41,7 @@ const Checkbox = (props) => { > - {info && } + {info && } diff --git a/webapp/components/form/checkbox.scss b/webapp/components/form/checkbox.scss index 72a86c2b50..343d9d4a40 100644 --- a/webapp/components/form/checkbox.scss +++ b/webapp/components/form/checkbox.scss @@ -1,4 +1,8 @@ .btn-checkbox { display: inline-flex; align-items: center; + + .info-icon-btn { + padding: 0 0.5rem; + } } diff --git a/webapp/views/App/views/Data/Data.js b/webapp/views/App/views/Data/Data.js index 22470e32ac..b8d0df0249 100644 --- a/webapp/views/App/views/Data/Data.js +++ b/webapp/views/App/views/Data/Data.js @@ -19,7 +19,7 @@ import ValidationReport from './ValidationReport' import Records from './Records' import Explorer from './Explorer' import Charts from './Charts' -import ExportData from './ExportData' +import DataExport from './DataExport' import DataImport from './DataImport' import { MapView } from './MapView' @@ -83,7 +83,7 @@ const Data = () => { ...(Authorizer.canExportRecords(user, surveyInfo) ? [ { - component: ExportData, + component: DataExport, path: dataModules.export.path, }, ] diff --git a/webapp/views/App/views/Data/ExportData/ExportData.js b/webapp/views/App/views/Data/DataExport/DataExport.js similarity index 93% rename from webapp/views/App/views/Data/ExportData/ExportData.js rename to webapp/views/App/views/Data/DataExport/DataExport.js index 9cfdf26c8a..8e2e2c42a3 100644 --- a/webapp/views/App/views/Data/ExportData/ExportData.js +++ b/webapp/views/App/views/Data/DataExport/DataExport.js @@ -1,4 +1,4 @@ -import './ExportData.scss' +import './DataExport.scss' import React, { useState } from 'react' import { useDispatch } from 'react-redux' @@ -20,6 +20,7 @@ const exportOptions = { includeCategoryItemsLabels: 'includeCategoryItemsLabels', expandCategoryItems: 'expandCategoryItems', includeCategories: 'includeCategories', + includeAncestorAttributes: 'includeAncestorAttributes', includeAnalysis: 'includeAnalysis', includeDataFromAllCycles: 'includeDataFromAllCycles', includeFiles: 'includeFiles', @@ -33,6 +34,7 @@ const defaultOptionsSelection = { [exportOptions.includeCategoryItemsLabels]: true, [exportOptions.expandCategoryItems]: false, [exportOptions.includeCategories]: false, + [exportOptions.includeAncestorAttributes]: false, [exportOptions.includeAnalysis]: false, [exportOptions.includeDataFromAllCycles]: false, [exportOptions.includeFiles]: false, @@ -44,7 +46,7 @@ const sources = { selectedRecords: 'selectedRecords', } -const ExportData = (props) => { +const DataExport = (props) => { const { recordUuids, search, sourceSelectionAvailable } = props const dispatch = useDispatch() @@ -104,6 +106,7 @@ const ExportData = (props) => { exportOptions.includeCategoryItemsLabels, exportOptions.expandCategoryItems, exportOptions.includeCategories, + exportOptions.includeAncestorAttributes, ...(canAnalyzeRecords ? [exportOptions.includeAnalysis] : []), ...(cycles.length > 1 ? [exportOptions.includeDataFromAllCycles] : []), exportOptions.includeFiles, @@ -128,14 +131,14 @@ const ExportData = (props) => { ) } -ExportData.propTypes = { +DataExport.propTypes = { recordUuids: PropTypes.array, search: PropTypes.string, sourceSelectionAvailable: PropTypes.bool, } -ExportData.defaultProps = { +DataExport.defaultProps = { sourceSelectionAvailable: false, } -export default ExportData +export default DataExport diff --git a/webapp/views/App/views/Data/ExportData/ExportData.scss b/webapp/views/App/views/Data/DataExport/DataExport.scss similarity index 100% rename from webapp/views/App/views/Data/ExportData/ExportData.scss rename to webapp/views/App/views/Data/DataExport/DataExport.scss diff --git a/webapp/views/App/views/Data/DataExport/index.js b/webapp/views/App/views/Data/DataExport/index.js new file mode 100644 index 0000000000..0022a198f8 --- /dev/null +++ b/webapp/views/App/views/Data/DataExport/index.js @@ -0,0 +1 @@ +export { default } from './DataExport' diff --git a/webapp/views/App/views/Data/ExportData/index.js b/webapp/views/App/views/Data/ExportData/index.js deleted file mode 100644 index 1b1c97ec95..0000000000 --- a/webapp/views/App/views/Data/ExportData/index.js +++ /dev/null @@ -1 +0,0 @@ -export { default } from './ExportData' diff --git a/webapp/views/App/views/Data/Records/HeaderLeft/RecordsDataExportModal.js b/webapp/views/App/views/Data/Records/HeaderLeft/RecordsDataExportModal.js index ab599db5ef..5be0cc8cbc 100644 --- a/webapp/views/App/views/Data/Records/HeaderLeft/RecordsDataExportModal.js +++ b/webapp/views/App/views/Data/Records/HeaderLeft/RecordsDataExportModal.js @@ -4,7 +4,7 @@ import React from 'react' import PropTypes from 'prop-types' import { Modal, ModalBody } from '@webapp/components/modal' -import ExportData from '../../ExportData' +import DataExport from '../../DataExport' export const RecordsDataExportModal = (props) => { const { onClose, recordUuids, search } = props @@ -12,7 +12,7 @@ export const RecordsDataExportModal = (props) => { return ( - + )