Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CSV data export: include option to export creation date #3603

Merged
merged 9 commits into from
Oct 21, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions common/model/db/tables/dataNodeDef/colUtils.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
import * as R from 'ramda'

// MOVE import from table
import * as NodeDefTable from '@common/surveyRdb/nodeDefTable'
import * as ColProps from './dataColProps'

const getValuesByColumnName = ({ survey, nodeDefCol, nodeCol = {} }) => {
const valueFnProcessor = ColProps.getColValueProcessor(nodeDefCol)
const valueFn = valueFnProcessor({ survey, nodeDefCol, nodeCol })
return NodeDefTable.getColumnNames(nodeDefCol).reduce((acc, columnName) => {
acc[columnName] = valueFn(nodeCol, columnName)
return acc
}, {})
}

const getValues = (survey, nodeDefCol, nodeCol = {}) => {
const valuesByColumnName = getValuesByColumnName({ survey, nodeDefCol, nodeCol })
return Object.values(valuesByColumnName)
}

const getValue = R.pipe(getValues, R.head)

export const TableDataNodeDefColUtils = {
getValue,
getValues,
getValuesByColumnName,
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
import * as camelize from 'camelize'

import { Objects, PointFactory, Points, Strings } from '@openforis/arena-core'

import * as A from '@core/arena'
Expand All @@ -13,15 +11,15 @@ import * as NodeRefData from '@core/record/nodeRefData'
import * as DateTimeUtils from '@core/dateUtils'

import * as NodeDefTable from '@common/surveyRdb/nodeDefTable'
import { ColumnNodeDef } from '@common/model/db'
import ColumnNodeDef from './columnNodeDef'

const { nodeDefType } = NodeDef

const colValueProcessor = 'colValueProcessor'

const getValueFromItem = (nodeDefCol, columnName, item = {}, isInProps = false) => {
// Remove nodeDefName from col name
const prop = camelize(NodeDefTable.extractColumnName(nodeDefCol, columnName))
const prop = A.camelize(NodeDefTable.extractColumnName(nodeDefCol, columnName))

return isInProps ? NodeDef.getProp(prop)(item) : A.propOr(null, prop, item)
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import * as R from 'ramda'

import * as NodeDef from '@core/survey/nodeDef'
import * as DataCol from './dataCol'
import { TableDataNodeDefColUtils } from './colUtils'

const getNodeCol = (nodeDefCol, nodeRow) => {
const nodeDefUuidCol = NodeDef.getUuid(nodeDefCol)
Expand All @@ -11,12 +11,14 @@ const getNodeCol = (nodeDefCol, nodeRow) => {
return nodeDefUuidRow === nodeDefUuidCol ? nodeRow : R.pathOr({}, ['children', nodeDefUuidCol], nodeRow)
}

export const getValues = ({ survey, nodeRow, nodeDefColumns }) => {
const values = nodeDefColumns.map((nodeDefCol) => {
const getValuesByColumnName = ({ survey, nodeRow, nodeDefColumns }) =>
nodeDefColumns.reduce((acc, nodeDefCol) => {
const nodeCol = getNodeCol(nodeDefCol, nodeRow)
const nodeColValues = DataCol.getValues(survey, nodeDefCol, nodeCol)
return nodeColValues
})
const valuesByColumnName = TableDataNodeDefColUtils.getValuesByColumnName({ survey, nodeDefCol, nodeCol })
Object.assign(acc, valuesByColumnName)
return acc
}, {})

return R.flatten(values)
export const TableDataNodeDefRowUtils = {
getValuesByColumnName,
}
94 changes: 71 additions & 23 deletions common/model/db/tables/dataNodeDef/table.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,19 +7,43 @@ import Table from '../table'
import TableSurveyRdb from '../tableSurveyRdb'
import TableRecord from '../record'
import ColumnNodeDef from './columnNodeDef'
import { TableDataNodeDefRowUtils } from './rowUtils'

const columnSet = {
id: Table.columnSetCommon.id,
dateCreated: Table.columnSetCommon.dateCreated,
dateModified: Table.columnSetCommon.dateModified,
uuid: Table.columnSetCommon.uuid,
parentUuid: 'parent_uuid',
ancestorUuid: 'ancestor_uuid',
recordUuid: 'record_uuid',
recordCycle: 'record_cycle',
recordStep: 'record_step',
recordOwnerUuid: 'record_owner_uuid',
}

const rootDefColumnNames = [
columnSet.recordUuid,
columnSet.recordCycle,
columnSet.recordStep,
columnSet.recordOwnerUuid,
]

const commonColumnNamesAndTypes = [
`${columnSet.id} bigint NOT NULL GENERATED ALWAYS AS IDENTITY`,
`${columnSet.dateCreated} TIMESTAMP NOT NULL DEFAULT (now() AT TIME ZONE 'UTC')`,
`${columnSet.dateModified} TIMESTAMP NOT NULL DEFAULT (now() AT TIME ZONE 'UTC')`,
`${columnSet.uuid} uuid NOT NULL`,
`${columnSet.parentUuid} uuid NULL`,
]

const rootDefColumnNamesAndTypes = [
`${columnSet.recordUuid} uuid NOT NULL`,
`${columnSet.recordCycle} varchar(2) NOT NULL`,
`${columnSet.recordStep} varchar(63) NOT NULL`,
`${columnSet.recordOwnerUuid} uuid NOT NULL`,
]

/**
* @typedef {module:arena.TableSurveyRdb} module:arena.TableDataNodeDef
* @property {Survey} survey - The survey.
Expand Down Expand Up @@ -69,36 +93,41 @@ export default class TableDataNodeDef extends TableSurveyRdb {
return this.getColumn(columnSet.recordOwnerUuid)
}

getNodeDefsForColumns({ includeAnalysis = true } = {}) {
const { nodeDef, survey } = this
if (NodeDef.isAttribute(nodeDef)) {
// Multiple attr table
return [nodeDef]
}
return R.pipe(
Survey.getNodeDefDescendantAttributesInSingleEntities({ nodeDef, includeAnalysis }),
R.filter(NodeDef.isSingleAttribute),
R.sortBy(R.ascend(R.prop('id')))
)(survey)
}

get columnNodeDefs() {
let nodeDefs = null
if (NodeDef.isAttribute(this.nodeDef)) {
nodeDefs = [this.nodeDef] // Multiple attr table
} else {
const descendants = Survey.getNodeDefDescendantAttributesInSingleEntities({
nodeDef: this.nodeDef,
includeAnalysis: true,
})(this.survey)
nodeDefs = R.sortBy(R.ascend(R.prop('id')))(descendants)
return this.getNodeDefsForColumns({ includeAnalysis: true }).map(
(nodeDefColumn) => new ColumnNodeDef(this, nodeDefColumn)
)
}

getColumnNames = ({ includeAnalysis = true } = {}) => {
const { nodeDef } = this
const nodeDefsForColumns = this.getNodeDefsForColumns({ includeAnalysis })
const names = [columnSet.uuid, columnSet.parentUuid]
if (NodeDef.isRoot(nodeDef)) {
names.push(...rootDefColumnNames)
}
return nodeDefs.map((nodeDefColumn) => new ColumnNodeDef(this, nodeDefColumn))
names.push(...nodeDefsForColumns.flatMap((nodeDefCol) => ColumnNodeDef.getColumnNames(nodeDefCol)))
return names
}

getColumnsWithType() {
const columnsAndType = []
columnsAndType.push(
`${columnSet.id} bigint NOT NULL GENERATED ALWAYS AS IDENTITY`,
`${columnSet.dateCreated} TIMESTAMP NOT NULL DEFAULT (now() AT TIME ZONE 'UTC')`,
`${columnSet.dateModified} TIMESTAMP NOT NULL DEFAULT (now() AT TIME ZONE 'UTC')`,
`${columnSet.uuid} uuid NOT NULL`,
`${columnSet.parentUuid} uuid NULL`
)
columnsAndType.push(...commonColumnNamesAndTypes)
if (NodeDef.isRoot(this.nodeDef)) {
columnsAndType.push(
`${columnSet.recordUuid} uuid NOT NULL`,
`${columnSet.recordCycle} varchar(2) NOT NULL`,
`${columnSet.recordStep} varchar(63) NOT NULL`,
`${columnSet.recordOwnerUuid} uuid NOT NULL`
)
columnsAndType.push(...rootDefColumnNamesAndTypes)
}
this.columnNodeDefs.forEach((nodeDefColumn) => {
columnsAndType.push(...nodeDefColumn.names.map((name, i) => `${name} ${nodeDefColumn.types[i]}`))
Expand Down Expand Up @@ -131,6 +160,25 @@ export default class TableDataNodeDef extends TableSurveyRdb {
getConstraintUuidUnique() {
return `CONSTRAINT ${NodeDef.getName(this.nodeDef)}_uuid_unique_ix1 UNIQUE (${columnSet.uuid})`
}

getRowValuesByColumnName = ({ nodeRow, nodeDefColumns }) => {
const { survey, nodeDef } = this
const getRowValuesByColumnName = TableDataNodeDefRowUtils.getValuesByColumnName({ survey, nodeRow, nodeDefColumns })
return {
[columnSet.uuid]: nodeRow[columnSet.uuid],
[columnSet.parentUuid]: nodeRow[columnSet.ancestorUuid],
[columnSet.dateCreated]: nodeRow[columnSet.dateCreated],
[columnSet.dateModified]: nodeRow[columnSet.dateModified],
...(NodeDef.isRoot(nodeDef)
? rootDefColumnNames.reduce((acc, colName) => {
acc[colName] = nodeRow[colName]
return acc
}, {})
: {}),
...getRowValuesByColumnName,
}
}
}

TableDataNodeDef.columnSet = columnSet
TableDataNodeDef.rootDefColumnNames = rootDefColumnNames
3 changes: 2 additions & 1 deletion common/model/db/views/dataNodeDef/view.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ import * as NodeDef from '../../../../../core/survey/nodeDef'
import * as NodeDefExpression from '../../../../../core/survey/nodeDefExpression'
import * as Expression from '../../../../../core/expressionParser/expression'

import TableDataNodeDef, { ColumnNodeDef } from '../../tables/dataNodeDef'
import TableDataNodeDef from '../../tables/dataNodeDef'
import ColumnNodeDef from '../../tables/dataNodeDef/columnNodeDef'

const columnSet = {
dateCreated: TableDataNodeDef.columnSetCommon.dateCreated,
Expand Down
2 changes: 1 addition & 1 deletion common/surveyRdb/nodeDefTable.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import * as R from 'ramda'
import * as toSnakeCase from 'to-snake-case'

import * as NodeDef from '@core/survey/nodeDef'
import { ColumnNodeDef } from '@common/model/db'
import ColumnNodeDef from '@common/model/db/tables/dataNodeDef/columnNodeDef'

const viewSuffix = '_view'
const tablePrefix = 'data_'
Expand Down
1 change: 1 addition & 0 deletions core/i18n/resources/en/common.js
Original file line number Diff line number Diff line change
Expand Up @@ -559,6 +559,7 @@ $t(common.cantUndoWarning)`,
includeAncestorAttributes: 'Include ancestor attributes',
includeAnalysis: 'Include result variables',
includeDataFromAllCycles: 'Include data from all cycles',
includeDateCreated: 'Include creation date',
includeFiles: 'Include files',
includeFileAttributeDefs: 'Include file attribute columns',
includeInternalUuids: 'Include internal UUIDs',
Expand Down
5 changes: 2 additions & 3 deletions server/db/dbUtils.js
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@ export const selectDate = (field, fieldAlias = null) =>

export const now = "timezone('UTC', now())"

export const insertAllQueryBatch = (schema, table, cols, itemsValues) => {
export const insertAllQueryBatch = (schema, table, cols, valuesByColumnName) => {
const columnSet = new pgp.helpers.ColumnSet(cols, {
table: { schema, table },
})
return pgp.helpers.insert(itemsValues, columnSet)
return pgp.helpers.insert(valuesByColumnName, columnSet)
}

export const insertAllQuery = (schema, table, cols, itemsValues) => {
Expand All @@ -26,7 +26,6 @@ export const insertAllQuery = (schema, table, cols, itemsValues) => {
for (const [i, element] of cols.entries()) {
item[element] = itemValues[i]
}

return item
})

Expand Down
11 changes: 1 addition & 10 deletions server/modules/dataExport/service/dataExportService.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,7 @@ import * as JobManager from '@server/job/jobManager'
import DataExportJob from '@server/modules/dataExport/service/DataExportJob'

export const startCsvDataExportJob = ({ user, surveyId, cycle, recordUuids, search, options }) => {
const job = new DataExportJob({
user,
surveyId,
cycle,
recordUuids,
search,
options,
})

const job = new DataExportJob({ user, surveyId, cycle, recordUuids, search, options })
JobManager.executeJobThread(job)

return job
}
30 changes: 15 additions & 15 deletions server/modules/surveyRdb/manager/surveyRdbCsvExport.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,17 @@ import { Query } from '@common/model/query'
import { CsvDataExportModel } from '@common/model/csvExport'
import { ColumnNodeDef, ViewDataNodeDef } from '@common/model/db'

import * as DataTable from '@server/modules/surveyRdb/schemaRdb/dataTable'

const maxExpandedCategoryItems = 20

const csvObjectTransformerNullsToEmpty = (obj) => {
Object.entries(obj).forEach(([key, value]) => {
if (A.isNull(value)) {
obj[key] = ''
}
})
return obj
}

const visitCategoryItems = ({ survey, nodeDef, itemVisitor }) => {
const items = Survey.getNodeDefCategoryItems(nodeDef)(survey)
if (items.length <= maxExpandedCategoryItems) {
Expand Down Expand Up @@ -68,6 +75,7 @@ const getCsvExportFields = ({
includeCategoryItemsLabels = true,
expandCategoryItems = false,
includeInternalUuids = false,
includeDateCreated = false,
}) => {
const entityDef = Survey.getNodeDefByUuid(Query.getEntityDefUuid(query))(survey)
const viewDataNodeDef = new ViewDataNodeDef(survey, entityDef)
Expand All @@ -82,7 +90,10 @@ const getCsvExportFields = ({
}
if (addCycle) {
// Cycle is 0-based
fields.push(DataTable.columnNameRecordCycle)
fields.push(ViewDataNodeDef.columnSet.recordCycle)
}
if (includeDateCreated) {
fields.push(ViewDataNodeDef.columnSet.dateCreated)
}
fields.push(
...nodeDefCols.flatMap((nodeDefCol) => {
Expand Down Expand Up @@ -151,24 +162,13 @@ const getCsvObjectTransformerExpandCategoryItems = ({ survey, query }) => {
}
}

const getCsvObjectTransformerNullsToEmpty = () => {
return (obj) => {
Object.entries(obj).forEach(([key, value]) => {
if (A.isNull(value)) {
obj[key] = ''
}
})
return obj
}
}

const getCsvObjectTransformer = ({ survey, query, expandCategoryItems, nullsToEmpty = false }) => {
const transformers = []
if (expandCategoryItems) {
transformers.push(getCsvObjectTransformerExpandCategoryItems({ survey, query }))
}
if (nullsToEmpty) {
transformers.push(getCsvObjectTransformerNullsToEmpty())
transformers.push(csvObjectTransformerNullsToEmpty)
}
return transformers.length === 0 ? null : A.pipe(...transformers)
}
Expand Down
5 changes: 5 additions & 0 deletions server/modules/surveyRdb/manager/surveyRdbManager.js
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ export const fetchViewData = async (params, client = db) => {
includeCategoryItemsLabels = true,
expandCategoryItems = false,
includeInternalUuids = false,
includeDateCreated = false,
nullsToEmpty = false,
} = params

Expand All @@ -90,6 +91,7 @@ export const fetchViewData = async (params, client = db) => {
query,
columnNodeDefs,
includeFileAttributeDefs,
includeDateCreated,
recordSteps,
recordOwnerUuid,
offset,
Expand All @@ -109,6 +111,7 @@ export const fetchViewData = async (params, client = db) => {
includeCategoryItemsLabels,
expandCategoryItems,
includeInternalUuids,
includeDateCreated,
})
await db.stream(result, (dbStream) => {
const csvTransform = CSVWriter.transformJsonToCsv({
Expand Down Expand Up @@ -213,6 +216,7 @@ export const fetchEntitiesDataToCsvFiles = async (
includeFileAttributeDefs,
includeFiles,
includeInternalUuids,
includeDateCreated,
recordsModifiedAfter,
} = options

Expand Down Expand Up @@ -290,6 +294,7 @@ export const fetchEntitiesDataToCsvFiles = async (
includeCategoryItemsLabels,
expandCategoryItems,
includeInternalUuids,
includeDateCreated,
},
client
)
Expand Down
Loading
Loading