Feature: file name expression (#3662)

* file name expression (WIP)

* update fileName on node update

* show calculated file name in ui

* extract file name calcualted from data

* prepare unique file names generator

* prepare unique names generator

* generate unique file names in data export

* code cleanup

* csv data extraction job: keep track of assigned file names

* fixed unique file names generator

* import fileNameExpression from Collect surveys

* file name expression: use expression editor (WIP)

* layout adjustments

* code cleanup

* attribute file download: use generated file name (if any)

* code cleanup

* update calculated file name on survey publish

* fixed SonarCloud issue

* code cleanup

---------

Co-authored-by: Stefano Ricci <[email protected]>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

common/model/db/tables/dataNodeDef/columnNodeDef.js

@@ -9,6 +9,8 @@ import * as SQL from '../../sql'
 const { nodeDefType } = NodeDef
 
 const columnSuffixCodeLabel = '_label'
+const columnSuffixFileUuid = '_file_uuid'
+const columnSuffixFileName = '_file_name'
 const columnSuffixTaxonScientificName = '_scientific_name'
 const columnSuffixTaxonVernacularName = '_vernacular_name'
 
@@ -21,7 +23,7 @@ const columnNamesSuffixGetterByType = {
     return suffixes
   },
   [nodeDefType.taxon]: () => ['', columnSuffixTaxonScientificName, columnSuffixTaxonVernacularName],
-  [nodeDefType.file]: () => ['_file_uuid', '_file_name'],
+  [nodeDefType.file]: () => [columnSuffixFileUuid, columnSuffixFileName],
 }
 
 const colTypesGetterByType = {
@@ -109,16 +111,24 @@ export default class ColumnNodeDef {
   }
 
   get codeLabelColumn() {
-    if (!NodeDef.isCode(this.nodeDef)) return null
-    return `${NodeDef.getName(this.nodeDef)}${columnSuffixCodeLabel}`
+    return NodeDef.isCode(this.nodeDef) ? `${NodeDef.getName(this.nodeDef)}${columnSuffixCodeLabel}` : null
+  }
+
+  get fileNameColumn() {
+    return NodeDef.isFile(this.nodeDef) ? `${NodeDef.getName(this.nodeDef)}${columnSuffixFileName}` : null
   }
 }
 
 ColumnNodeDef.columnSuffixCodeLabel = columnSuffixCodeLabel
+ColumnNodeDef.columnSuffixFileUuid = columnSuffixFileUuid
+ColumnNodeDef.columnSuffixFileName = columnSuffixFileName
 ColumnNodeDef.columnSuffixTaxonScientificName = columnSuffixTaxonScientificName
 ColumnNodeDef.columnSuffixTaxonVernacularName = columnSuffixTaxonVernacularName
 
-ColumnNodeDef.getCodeLabelColumnName = (nodeDef) => `${NodeDef.getName(nodeDef)}${columnSuffixCodeLabel}`
+const getColumnNameWithSuffix = (suffix) => (nodeDef) => `${NodeDef.getName(nodeDef)}${suffix}`
+ColumnNodeDef.getCodeLabelColumnName = getColumnNameWithSuffix(columnSuffixCodeLabel)
+ColumnNodeDef.getFileNameColumnName = getColumnNameWithSuffix(columnSuffixFileName)
+ColumnNodeDef.getFileUuidColumnName = getColumnNameWithSuffix(columnSuffixFileUuid)
 
 ColumnNodeDef.getColumnNames = getColumnNames
 ColumnNodeDef.getColumnName = R.pipe(ColumnNodeDef.getColumnNames, R.head)

common/model/db/tables/dataNodeDef/dataColProps.js

@@ -1,29 +1,21 @@
 import { Objects, PointFactory, Points, Strings } from '@openforis/arena-core'
 
 import * as A from '@core/arena'
+import * as DateTimeUtils from '@core/dateUtils'
 import * as NumberUtils from '@core/numberUtils'
-import * as Survey from '@core/survey/survey'
+import * as Node from '@core/record/node'
+import * as NodeRefData from '@core/record/nodeRefData'
 import * as CategoryItem from '@core/survey/categoryItem'
 import * as NodeDef from '@core/survey/nodeDef'
+import * as Survey from '@core/survey/survey'
 import * as Taxon from '@core/survey/taxon'
-import * as Node from '@core/record/node'
-import * as NodeRefData from '@core/record/nodeRefData'
-import * as DateTimeUtils from '@core/dateUtils'
 
-import * as NodeDefTable from '@common/surveyRdb/nodeDefTable'
 import ColumnNodeDef from './columnNodeDef'
 
 const { nodeDefType } = NodeDef
 
 const colValueProcessor = 'colValueProcessor'
 
-const getValueFromItem = (nodeDefCol, columnName, item = {}, isInProps = false) => {
-  // Remove nodeDefName from col name
-  const prop = A.camelize(NodeDefTable.extractColumnName(nodeDefCol, columnName))
-
-  return isInProps ? NodeDef.getProp(prop)(item) : A.propOr(null, prop, item)
-}
-
 const _extractCategoryItem = ({ survey, node }) => {
   let item = NodeRefData.getCategoryItem(node)
   if (item) return item
@@ -38,13 +30,6 @@ const _extractTaxon = ({ survey, node }) => {
   return NodeRefData.getTaxon(node) ?? Survey.getTaxonByUuid(taxonUuid)(survey)
 }
 
-const nodeValuePropProcessor =
-  ({ nodeDefCol }) =>
-  (node, columnName) => {
-    const nodeValue = Node.getValue(node)
-    return getValueFromItem(nodeDefCol, columnName, nodeValue)
-  }
-
 /**
  * Convert an input value to RDB compatible output value.
  * The contract is such that the value output value must always be compatible with RDB.
@@ -161,15 +146,24 @@ const props = {
   },
 
   [nodeDefType.file]: {
-    [colValueProcessor]: nodeValuePropProcessor,
+    [colValueProcessor]: ({ nodeDefCol }) => {
+      const fileNameExpression = NodeDef.getFileNameExpression(nodeDefCol)
+      return (node, columnName) => {
+        if (columnName.endsWith(ColumnNodeDef.columnSuffixFileName)) {
+          return fileNameExpression ? Node.getFileNameCalculated(node) : Node.getFileName(node)
+        }
+        if (columnName.endsWith(ColumnNodeDef.columnSuffixFileUuid)) {
+          return Node.getFileUuid(node)
+        }
+        return null
+      }
+    },
   },
 }
 
 export const getColValueProcessor = (nodeDef) =>
   A.propOr(
-    () => (node) => {
-      return Node.isValueBlank(node) ? null : Node.getValue(node)
-    },
+    () => (node) => (Node.isValueBlank(node) ? null : Node.getValue(node)),
     colValueProcessor,
     props[NodeDef.getType(nodeDef)]
   )

core/i18n/resources/en/common.js

@@ -1282,15 +1282,16 @@ E.g. this.region = region_attribute_name
       maxNumberDecimalDigits: 'Max number of decimal digits',
     },
     fileProps: {
-      numberOfFiles: 'Go to Validations to change the Min. and Max. number of files.',
-      maxFileSize: 'Max. file size (Mb)',
+      fileNameExpression: 'File name expression',
       fileType: 'File type',
       fileTypes: {
         image: 'Image',
         video: 'Video',
         audio: 'Audio',
         other: 'Other',
       },
+      maxFileSize: 'Max. file size (Mb)',
+      numberOfFiles: 'Go to Validations to change the Min. and Max. number of files.',
     },
     mobileProps: {
       title: 'Mobile App',

core/record/node.js

@@ -246,6 +246,7 @@ export const getDateModified = R.prop(keys.dateModified)
 
 // File
 export const getFileName = _getValuePropRaw(valuePropsFile.fileName, '')
+export const getFileNameCalculated = _getValuePropRaw(valuePropsFile.fileNameCalculated, '')
 export const getFileUuid = _getValuePropRaw(valuePropsFile.fileUuid)
 export const newNodeValueFile = ({ fileUuid, fileName }) => ({
   [valuePropsFile.fileUuid]: fileUuid,

core/record/recordFile.js

@@ -1,7 +1,9 @@
 import * as R from 'ramda'
 
-import { truncate } from '@core/stringUtils'
+import { FileNames } from '@openforis/arena-core'
+
 import * as ObjectUtils from '@core/objectUtils'
+import { truncate } from '@core/stringUtils'
 import { uuidv4 } from '@core/uuid'
 import * as Node from './node'
 
@@ -45,11 +47,9 @@ export const createFileFromNode = ({ node, size = null, content = null }) =>
     content,
   })
 
-const getExtensionFromFileName = (fileName) => R.pipe(R.split('.'), R.tail)(fileName)
-
 export const truncateFileName = (fileName, maxLength = 10) => {
   if (fileName && !R.isEmpty(fileName)) {
-    const extension = getExtensionFromFileName(fileName)
+    const extension = FileNames.getExtension(fileName)
 
     return R.pipe(R.dropLast(extension.length + 1), truncate(maxLength), (name) => `${name}.${extension}`)(fileName)
   }
@@ -65,7 +65,7 @@ export const getSize = ObjectUtils.getProp(propKeys.size)
 export const getNodeUuid = ObjectUtils.getProp(propKeys.nodeUuid)
 export const getRecordUuid = ObjectUtils.getProp(propKeys.recordUuid)
 export const getContent = R.prop(keys.content)
-export const getExtension = R.pipe(getName, getExtensionFromFileName)
+export const getExtension = R.pipe(getName, FileNames.getExtension)
 
 // UPDATE
 export const assocContent = R.assoc(keys.content)

core/survey/_surveyValidator/nodeDefValidator.js

@@ -13,7 +13,7 @@ import * as NodeDefLayout from '../nodeDefLayout'
 import * as NodeDefExpressionsValidator from './nodeDefExpressionsValidator'
 import * as NodeDefValidationsValidator from './nodeDefValidationsValidator'
 
-const { keys, propKeys } = NodeDef
+const { keys, keysPropsAdvanced, propKeys } = NodeDef
 
 const keysValidationFields = {
   children: 'children',
@@ -104,20 +104,32 @@ const validateVirtualEntityFormula = (survey, nodeDef) =>
     : null
 
 const validateItemsFilterExpression = (survey, nodeDef) => {
-  if (R.isEmpty(NodeDef.getItemsFilter(nodeDef))) return null
+  const expression = NodeDef.getItemsFilter(nodeDef)
+  if (R.isEmpty(expression)) return null
 
   const { validationResult } = nodeDefExpressionValidator.validate({
     survey,
     nodeDefCurrent: nodeDef,
-    expression: NodeDef.getItemsFilter(nodeDef),
+    expression,
     isContextParent: true,
     selfReferenceAllowed: true,
     itemsFilter: true,
   })
-  if (validationResult && !validationResult.valid) {
-    return Validation.newInstance(false, {}, [validationResult])
-  }
-  return null
+  return validationResult && !validationResult.valid ? Validation.newInstance(false, {}, [validationResult]) : null
+}
+
+const validateFileNameExpression = (survey, nodeDef) => {
+  const expression = NodeDef.getFileNameExpression(nodeDef)
+  if (R.isEmpty(expression)) return null
+
+  const { validationResult } = nodeDefExpressionValidator.validate({
+    survey,
+    nodeDefCurrent: nodeDef,
+    expression,
+    isContextParent: true,
+    selfReferenceAllowed: false,
+  })
+  return validationResult && !validationResult.valid ? Validation.newInstance(false, {}, [validationResult]) : null
 }
 
 const validateColumnWidth =
@@ -192,36 +204,31 @@ const propsValidations = (survey) => ({
 })
 
 const validateAdvancedProps = async (survey, nodeDef) => {
-  const [
-    validationDefaultValues,
-    validationApplicable,
-    validationValidations,
-    validationVirtualEntityFormula,
-    validationItemsFilter,
-  ] = await Promise.all([
-    NodeDefExpressionsValidator.validate(survey, nodeDef, Survey.dependencyTypes.defaultValues),
-    NodeDefExpressionsValidator.validate(survey, nodeDef, Survey.dependencyTypes.applicable),
-    NodeDefValidationsValidator.validate(survey, nodeDef),
-    validateVirtualEntityFormula(survey, nodeDef),
-    validateItemsFilterExpression(survey, nodeDef),
-  ])
-
-  return Validation.newInstance(
-    R.all(Validation.isValid, [
-      validationDefaultValues,
-      validationApplicable,
-      validationValidations,
-      validationVirtualEntityFormula,
-      validationItemsFilter,
-    ]),
-    R.reject(Validation.isValid, {
-      [NodeDef.keysPropsAdvanced.defaultValues]: validationDefaultValues,
-      [NodeDef.keysPropsAdvanced.applicable]: validationApplicable,
-      [NodeDef.keysPropsAdvanced.validations]: validationValidations,
-      [NodeDef.keysPropsAdvanced.formula]: validationVirtualEntityFormula,
-      [NodeDef.keysPropsAdvanced.itemsFilter]: validationItemsFilter,
-    })
-  )
+  const validatorsByProp = {
+    [keysPropsAdvanced.defaultValues]: NodeDefExpressionsValidator.validate(
+      survey,
+      nodeDef,
+      Survey.dependencyTypes.defaultValues
+    ),
+    [keysPropsAdvanced.applicable]: NodeDefExpressionsValidator.validate(
+      survey,
+      nodeDef,
+      Survey.dependencyTypes.applicable
+    ),
+    [keysPropsAdvanced.validations]: NodeDefValidationsValidator.validate(survey, nodeDef),
+    [keysPropsAdvanced.formula]: validateVirtualEntityFormula(survey, nodeDef),
+    [keysPropsAdvanced.itemsFilter]: validateItemsFilterExpression(survey, nodeDef),
+    [keysPropsAdvanced.fileNameExpression]: validateFileNameExpression(survey, nodeDef),
+  }
+  const validationResultsArray = await Promise.all(Object.values(validatorsByProp))
+  const validationResultsByProp = Object.keys(validatorsByProp).reduce((acc, prop, index) => {
+    acc[prop] = validationResultsArray[index]
+    return acc
+  }, {})
+  const valid = R.all(Validation.isValid, validationResultsArray)
+  const notValidValidationsByProp = R.reject(Validation.isValid, validationResultsByProp)
+
+  return Validation.newInstance(valid, notValidValidationsByProp)
 }
 
 export const validateNodeDef = async (survey, nodeDef) => {

core/survey/nodeDef.js

@@ -31,6 +31,7 @@ export const keys = {
   draftAdvanced: 'draftAdvanced',
   draftAdvancedApplicable: 'draftAdvancedApplicable',
   draftAdvancedDefaultValues: 'draftAdvancedDefaultValues',
+  draftAdvancedFileNameExpression: 'draftAdvancedFileNameExpression',
   draftAdvancedValidations: 'draftAdvancedValidations',
   type: 'type',
   deleted: 'deleted',
@@ -154,6 +155,8 @@ export const keysPropsAdvanced = {
 
   // code and taxon
   itemsFilter: 'itemsFilter',
+  // file
+  fileNameExpression: 'fileNameExpression',
 }
 
 const commonAttributePropsAdvancedKeys = [
@@ -367,6 +370,8 @@ export const getAllPropsAndAllPropsDraft =
 export const hasAdvancedPropsDraft = (nodeDef) => R.prop(keys.draftAdvanced, nodeDef) === true
 export const hasAdvancedPropsApplicableDraft = (nodeDef) => R.prop(keys.draftAdvancedApplicable, nodeDef) === true
 export const hasAdvancedPropsDefaultValuesDraft = (nodeDef) => R.prop(keys.draftAdvancedDefaultValues, nodeDef) === true
+export const hasAdvancedPropsFileNameExpressionDraft = (nodeDef) =>
+  R.prop(keys.draftAdvancedFileNameExpression, nodeDef) === true
 export const hasAdvancedPropsValidationsDraft = (nodeDef) => R.prop(keys.draftAdvancedValidations, nodeDef) === true
 const isPropAdvanced = (key) => Object.keys(keysPropsAdvanced).includes(key)
 
@@ -391,13 +396,16 @@ export const getAllExpressions = (nodeDef) => {
     return acc
   }, [])
   ArrayUtils.addIfNotEmpty(getItemsFilter(nodeDef))(expressions)
+  ArrayUtils.addIfNotEmpty(getFileNameExpression(nodeDef))(expressions)
   return expressions
 }
 
 export const isExcludedInClone = getPropAdvanced(keysPropsAdvanced.excludedInClone, false)
 
 // code and taxon
 export const getItemsFilter = getPropAdvanced(keysPropsAdvanced.itemsFilter, '')
+// file
+export const getFileNameExpression = getPropAdvanced(keysPropsAdvanced.fileNameExpression, '')
 
 // Advanced props - Analysis
 export const getFormula = getPropAdvanced(keysPropsAdvanced.formula, [])

core/survey/nodeValueProps.js

@@ -24,6 +24,7 @@ export const valuePropsDate = {
 export const valuePropsFile = {
   fileUuid: 'fileUuid',
   fileName: 'fileName',
+  fileNameCalculated: 'fileNameCalculated',
   fileSize: 'fileSize',
 }
 

server/modules/collectImport/service/collectImport/metaImportJobs/nodeDefsImportJob/nodeDefsImportJob.js

@@ -381,7 +381,12 @@ export default class NodeDefsImportJob extends Job {
         ]
       }
     }
-
+    if (type === NodeDef.nodeDefType.file) {
+      const fileNameExpression = CollectSurvey.getCollectAttribute('fileNameExpression')(collectNodeDef)
+      if (fileNameExpression) {
+        propsAdvanced[NodeDef.keysPropsAdvanced.fileNameExpression] = fileNameExpression
+      }
+    }
     return propsAdvanced
   }
 

server/modules/collectImport/service/collectImport/model/collectSurvey.js

@@ -142,7 +142,8 @@ export const getUiAttribute =
 export const getCollectAttribute =
   (name, defaultValue = null) =>
   (collectXmlElement) => {
-    const value = getAttribute(`collect:${name}`, defaultValue)(collectXmlElement)
+    const value =
+      getAttribute(`collect:${name}`)(collectXmlElement) ?? getAttribute(`n0:${name}`, defaultValue)(collectXmlElement)
     return _transformValue(value)
   }
 

server/modules/dataExport/service/DataExportJob/jobs/CSVDataExtractionJob.js

@@ -10,7 +10,7 @@ export default class CSVDataExtractionJob extends Job {
   async execute() {
     const { survey, cycle, recordUuids, search, options, outputDir } = this.context
 
-    await SurveyRdbService.fetchEntitiesDataToCsvFiles(
+    const { fileNamesByFileUuid } = await SurveyRdbService.fetchEntitiesDataToCsvFiles(
       {
         user: this.user,
         survey,
@@ -26,5 +26,6 @@ export default class CSVDataExtractionJob extends Job {
       },
       this.tx
     )
+    this.setContext({ fileNamesByFileUuid })
   }
 }