Records data export: allow exporting only filtered records (#3175)

* updated env.template file * records data export: allow exporting only filtered records --------- Co-authored-by: Stefano Ricci <[email protected]>
openforis · Dec 4, 2023 · e7da615 · e7da615
1 parent f99ba3e
commit e7da615
Show file tree

Hide file tree

Showing 12 changed files with 84 additions and 33 deletions.
diff --git a/.env.template b/.env.template
@@ -20,14 +20,14 @@ TEMP_FOLDER=/tmp/arena_upload
 # (if both FILE_STORAGE_PATH and FILE_STORAGE_AWS_S3_BUCKET_NAME are not specified, files will be stored in DB)
 
 # FILES STORAGE (file system)
-# path of a folder in the file system used to store files
-FILE_STORAGE_PATH=
+# path of a folder in the file system used to store files (uncomment the following line to enable file system files storage)
+# FILE_STORAGE_PATH=
 
-# FILES STORAGE (AWS S3 Bucket)
-FILE_STORAGE_AWS_S3_BUCKET_NAME=
-FILE_STORAGE_AWS_S3_BUCKET_REGION=
-FILE_STORAGE_AWS_ACCESS_KEY=
-FILE_STORAGE_AWS_SECRET_ACCESS_KEY=
+# FILES STORAGE (AWS S3 Bucket) (uncomment the following lines to enable S3 bucket files storage)
+# FILE_STORAGE_AWS_S3_BUCKET_NAME=
+# FILE_STORAGE_AWS_S3_BUCKET_REGION=
+# FILE_STORAGE_AWS_ACCESS_KEY=
+# FILE_STORAGE_AWS_SECRET_ACCESS_KEY=
 
 # Email
 SENDGRID_API_KEY=

diff --git a/core/i18n/resources/en.js b/core/i18n/resources/en.js
@@ -717,6 +717,7 @@ Are you sure you want to continue?`,
       source: {
         label: 'Source',
         allRecords: 'All records',
+        filteredRecords: 'Only filtered records',
         selectedRecord: 'Only selected record',
         selectedRecord_other: 'Only selected {{count}} records',
       },

diff --git a/server/modules/dataExport/api/dataExportApi.js b/server/modules/dataExport/api/dataExportApi.js
@@ -21,6 +21,7 @@ export const init = (app) => {
           surveyId,
           cycle,
           recordUuids,
+          search,
           includeCategories,
           includeCategoryItemsLabels,
           includeAnalysis,
@@ -35,6 +36,7 @@ export const init = (app) => {
           surveyId,
           cycle,
           recordUuids,
+          search,
           includeCategories,
           includeCategoryItemsLabels,
           includeAnalysis,

diff --git a/server/modules/survey/service/export/jobs/CSVDataExtractionJob.js b/server/modules/survey/service/export/jobs/CSVDataExtractionJob.js
@@ -12,6 +12,7 @@ export default class CSVDataExtractionJob extends Job {
       survey,
       cycle,
       recordUuids,
+      search,
       includeCategoryItemsLabels,
       includeAnalysis,
       includeDataFromAllCycles,
@@ -24,6 +25,7 @@ export default class CSVDataExtractionJob extends Job {
         survey,
         cycle,
         recordUuids,
+        search,
         includeCategoryItemsLabels,
         includeAnalysis,
         includeDataFromAllCycles,

diff --git a/server/modules/survey/service/surveyService.js b/server/modules/survey/service/surveyService.js
@@ -54,6 +54,7 @@ export const startExportCsvDataJob = ({
   surveyId,
   cycle,
   recordUuids,
+  search,
   includeCategories,
   includeCategoryItemsLabels,
   includeAnalysis,
@@ -65,6 +66,7 @@ export const startExportCsvDataJob = ({
     surveyId,
     cycle,
     recordUuids,
+    search,
     includeCategories,
     includeCategoryItemsLabels,
     includeAnalysis,

diff --git a/server/modules/surveyRdb/manager/surveyRdbManager.js b/server/modules/surveyRdb/manager/surveyRdbManager.js
@@ -1,11 +1,15 @@
 import pgPromise from 'pg-promise'
 
+import { Objects } from '@openforis/arena-core'
+
 import * as Survey from '@core/survey/survey'
 import * as NodeDef from '@core/survey/nodeDef'
+import * as Record from '@core/record/record'
 import * as PromiseUtils from '@core/promiseUtils'
 
 import * as FileUtils from '@server/utils/file/fileUtils'
 import * as DataTable from '@server/modules/surveyRdb/schemaRdb/dataTable'
+import * as RecordRepository from '@server/modules/record/repository/recordRepository'
 
 import { db } from '../../../db/db'
 import * as CSVWriter from '../../../utils/file/csvWriter'
@@ -199,7 +203,8 @@ export const fetchEntitiesDataToCsvFiles = async (
     includeCategoryItemsLabels,
     includeAnalysis,
     recordOwnerUuid = null,
-    recordUuids = null,
+    recordUuids: recordUuidsParam = null,
+    search = null,
     outputDir,
     callback,
   },
@@ -211,6 +216,23 @@ export const fetchEntitiesDataToCsvFiles = async (
     (nodeDef) => NodeDef.isRoot(nodeDef) || NodeDef.isMultiple(nodeDef)
   )
 
+  let recordUuids = null
+  if (recordUuidsParam) {
+    recordUuids = recordUuidsParam
+  } else if (!Objects.isEmpty(search)) {
+    const surveyId = Survey.getId(survey)
+    const nodeDefRoot = Survey.getNodeDefRoot(survey)
+    const nodeDefKeys = Survey.getNodeDefRootKeys(survey)
+    const recordsSummaries = await RecordRepository.fetchRecordsSummaryBySurveyId({
+      surveyId,
+      nodeDefRoot,
+      nodeDefKeys,
+      cycle,
+      search,
+    })
+    recordUuids = recordsSummaries.length > 0 ? recordsSummaries.map(Record.getUuid) : null
+  }
+
   callback?.({ total: nodeDefs.length })
 
   await PromiseUtils.each(nodeDefs, async (nodeDefContext, idx) => {

diff --git a/server/modules/surveyRdb/service/surveyRdbService.js b/server/modules/surveyRdb/service/surveyRdbService.js
@@ -137,6 +137,7 @@ export const fetchEntitiesDataToCsvFiles = async ({
   survey,
   cycle: cycleParam,
   recordUuids,
+  search,
   includeCategoryItemsLabels,
   includeAnalysis,
   includeDataFromAllCycles,
@@ -152,6 +153,7 @@ export const fetchEntitiesDataToCsvFiles = async ({
     cycle,
     outputDir,
     recordUuids,
+    search,
     includeCategoryItemsLabels,
     includeAnalysis,
     recordOwnerUuid,

diff --git a/webapp/service/api/data/index.js b/webapp/service/api/data/index.js
@@ -80,8 +80,13 @@ export const getDataImportFromCsvTemplatesUrl = ({ surveyId, cycle }) => {
 }
 
 // ==== DATA EXPORT
-export const startExportDataToCSVJob = async ({ surveyId, cycle, recordUuids, options }) => {
-  const { data } = await axios.post(`/api/survey/${surveyId}/data-export/csv`, { cycle, recordUuids, ...options })
+export const startExportDataToCSVJob = async ({ surveyId, cycle, recordUuids, search, options }) => {
+  const { data } = await axios.post(`/api/survey/${surveyId}/data-export/csv`, {
+    cycle,
+    recordUuids,
+    search,
+    ...options,
+  })
   const { job } = data
   return job
 }

diff --git a/webapp/store/ui/exportCsvData/actions.js b/webapp/store/ui/exportCsvData/actions.js
@@ -10,13 +10,13 @@ import { TestId } from '@webapp/utils/testId'
 import * as API from '@webapp/service/api'
 
 export const startCSVExport =
-  ({ recordUuids, options }) =>
+  ({ recordUuids, search, options }) =>
   async (dispatch, getState) => {
     const state = getState()
     const surveyId = SurveyState.getSurveyId(state)
     const cycle = SurveyState.getSurveyCycleKey(state)
 
-    const job = await API.startExportDataToCSVJob({ surveyId, cycle, recordUuids, options })
+    const job = await API.startExportDataToCSVJob({ surveyId, cycle, recordUuids, search, options })
 
     dispatch(
       JobActions.showJobMonitor({

diff --git a/webapp/views/App/views/Data/ExportData/ExportData.js b/webapp/views/App/views/Data/ExportData/ExportData.js
@@ -4,6 +4,8 @@ import React, { useState } from 'react'
 import { useDispatch } from 'react-redux'
 import PropTypes from 'prop-types'
 
+import { Objects } from '@openforis/arena-core'
+
 import { TestId } from '@webapp/utils/testId'
 
 import { ExportCsvDataActions } from '@webapp/store/ui'
@@ -32,11 +34,12 @@ const defaultOptionsSelection = {
 
 const sources = {
   allRecords: 'allRecords',
+  filteredRecords: 'filteredRecords',
   selectedRecords: 'selectedRecords',
 }
 
 const ExportData = (props) => {
-  const { sourceSelectionAvailable, recordUuids } = props
+  const { recordUuids, search, sourceSelectionAvailable } = props
 
   const dispatch = useDispatch()
   const i18n = useI18n()
@@ -58,30 +61,36 @@ const ExportData = (props) => {
     dispatch(
       ExportCsvDataActions.startCSVExport({
         recordUuids: source === sources.selectedRecords ? recordUuids : null,
+        search: source === sources.filteredRecords ? search : null,
         options: selectedOptions,
       })
     )
 
+  const availableSources = [
+    {
+      key: sources.allRecords,
+      label: `dataView.dataExport.source.allRecords`,
+    },
+  ]
+  if (sourceSelectionAvailable && recordUuids.length > 0) {
+    availableSources.push({
+      key: sources.selectedRecords,
+      label: `dataView.dataExport.source.selectedRecord`,
+      labelParams: { count: recordUuids.length },
+    })
+  }
+  if (sourceSelectionAvailable && !Objects.isEmpty(search)) {
+    availableSources.push({
+      key: sources.filteredRecords,
+      label: `dataView.dataExport.source.filteredRecords`,
+    })
+  }
+
   return (
     <div className="export">
-      {sourceSelectionAvailable && recordUuids.length > 0 && (
+      {availableSources.length > 1 && (
         <FormItem className="source-form-item" label={i18n.t('dataView.dataExport.source.label')}>
-          <RadioButtonGroup
-            onChange={onSourceChange}
-            value={source}
-            items={[
-              {
-                key: sources.allRecords,
-                label: `dataView.dataExport.source.allRecords`,
-              },
-              {
-                key: sources.selectedRecords,
-                label: `dataView.dataExport.source.selectedRecord`,
-                labelParams: { count: recordUuids.length },
-                disabled: recordUuids.length === 0,
-              },
-            ]}
-          />
+          <RadioButtonGroup onChange={onSourceChange} value={source} items={availableSources} />
         </FormItem>
       )}
       <ExpansionPanel className="options" buttonLabel="dataExportView.options.header">
@@ -113,6 +122,7 @@ const ExportData = (props) => {
 
 ExportData.propTypes = {
   recordUuids: PropTypes.array,
+  search: PropTypes.string,
   sourceSelectionAvailable: PropTypes.bool,
 }
 

diff --git a/webapp/views/App/views/Data/Records/HeaderLeft/HeaderLeft.js b/webapp/views/App/views/Data/Records/HeaderLeft/HeaderLeft.js
@@ -133,7 +133,11 @@ const HeaderLeft = ({ handleSearch, navigateToRecord, onRecordsUpdate, search, s
         <RecordsCloneModal onClose={toggleRecordsCloneModalOpen} selectedRecordsUuids={selectedRecordsUuids} />
       )}
       {recordsDataExportModalOpen && (
-        <RecordsDataExportModal onClose={toggleRecordsDataExportModalOpen} recordUuids={selectedRecordsUuids} />
+        <RecordsDataExportModal
+          onClose={toggleRecordsDataExportModalOpen}
+          recordUuids={selectedRecordsUuids}
+          search={search}
+        />
       )}
     </div>
   )

diff --git a/webapp/views/App/views/Data/Records/HeaderLeft/RecordsDataExportModal.js b/webapp/views/App/views/Data/Records/HeaderLeft/RecordsDataExportModal.js
@@ -7,12 +7,12 @@ import { Modal, ModalBody } from '@webapp/components/modal'
 import ExportData from '../../ExportData'
 
 export const RecordsDataExportModal = (props) => {
-  const { onClose, recordUuids } = props
+  const { onClose, recordUuids, search } = props
 
   return (
     <Modal className="records-data-export" onClose={onClose} showCloseButton title="dataView.dataExport.title">
       <ModalBody>
-        <ExportData sourceSelectionAvailable recordUuids={recordUuids} />
+        <ExportData search={search} recordUuids={recordUuids} sourceSelectionAvailable />
       </ModalBody>
     </Modal>
   )
@@ -21,4 +21,5 @@ export const RecordsDataExportModal = (props) => {
 RecordsDataExportModal.propTypes = {
   onClose: PropTypes.func.isRequired,
   recordUuids: PropTypes.array,
+  search: PropTypes.string,
 }