Fix CSV data import: date and time format check

core/i18n/resources/en/common.js

@@ -36,7 +36,7 @@ Do you want to ignore them?`,
     convert: 'Convert',
     copy: 'Copy',
     createdWith: 'Created with',
-    createdWithApp: `t(common.createdWith) {{app}}`,
+    createdWithApp: `$t(common.createdWith) {{app}}`,
     csvExport: 'CSV Export',
     csvImport: 'CSV Import',
     cycle: 'Cycle',
@@ -1665,6 +1665,8 @@ Levels will be renamed into level_1, level_2... level_N and an extra 'area' prop
         'Invalid date in column {{headers}}: {{value}}. Dates should be formatted as YYYY-MM-DD or DD/MM/YYYY. E.g. 2023-01-15 or 15/01/2023',
       invalidNumber: 'Invalid number in column {{headers}}: {{value}}',
       invalidTaxonCode: 'Invalid code in column {{headers}}: {{value}}',
+      invalidTime:
+        'Invalid time in column {{headers}}: {{value}}. Time should be formatted as HH:mm. E.g. 09:45 or 16:30',
       missingRequiredHeaders: 'Missing required columns: {{missingRequiredHeaders}}',
       errorUpdatingValues: 'Error updating values',
       multipleRecordsMatchingKeys: 'Multiple records found matching keys "{{keyValues}}"',

package.json

@@ -108,7 +108,7 @@
     "@mui/x-data-grid": "^7.14.0",
     "@mui/x-date-pickers": "^7.14.0",
     "@mui/x-tree-view": "^7.7.1",
-    "@openforis/arena-core": "^1.0.1",
+    "@openforis/arena-core": "^1.0.2",
     "@openforis/arena-server": "^0.1.36",
     "@reduxjs/toolkit": "^2.2.5",
     "@sendgrid/mail": "^8.1.3",

server/modules/dataImport/service/DataImportJob/dataImportCsvFileReader.js

@@ -8,21 +8,22 @@ import * as NodeDef from '@core/survey/nodeDef'
 import * as Category from '@core/survey/category'
 import * as Node from '@core/record/node'
 import * as DateUtils from '@core/dateUtils'
+import { uuidv4 } from '@core/uuid'
 import * as CSVReader from '@server/utils/file/csvReader'
 import * as FileUtils from '@server/utils/file/fileUtils'
-import { uuidv4 } from '@core/uuid'
 
 const VALUE_PROP_DEFAULT = 'value'
 
+const allowedBooleanValues = ['true', 'false', '1', '0']
 const allowedDateFormats = [
-  DateUtils.formats.dateISO,
-  'YYYY.MM.DD',
-  'YYYY/MM/DD',
   DateUtils.formats.dateDefault,
   'DD-MM-YYYY',
   'DD.MM.YYYY',
+  'YYYY.MM.DD',
+  'YYYY/MM/DD',
+  DateUtils.formats.dateISO,
 ]
-const allowedBooleanValues = ['true', 'false', '1', '0']
+const allowedTimeFormats = [DateUtils.formats.timeStorage]
 
 const singlePropValueConverter = ({ value }) => value[VALUE_PROP_DEFAULT]
 
@@ -35,6 +36,21 @@ const numericValueConverter = ({ value, headers }) => {
   return numericVal
 }
 
+const extractDateOrTime = ({ value, allowedFormats, formatTo, headers, errorKey }) => {
+  const val = singlePropValueConverter({ value })
+  let dateObj = null
+  const valueIsInValidFormat = allowedFormats.some((format) => {
+    // date and time values are without timezone
+    // use strict parsing to check if date is in the specified format
+    dateObj = DateUtils.parse(val, format, { keepTimeZone: false, strict: true })
+    return DateUtils.isValidDateObject(dateObj)
+  })
+  if (!valueIsInValidFormat) {
+    throw new SystemError(errorKey, { headers, value: val })
+  }
+  return DateUtils.format(dateObj, formatTo)
+}
+
 const valueConverterByNodeDefType = {
   [NodeDef.nodeDefType.boolean]: ({ value, headers }) => {
     const val = singlePropValueConverter({ value })
@@ -64,18 +80,14 @@ const valueConverterByNodeDefType = {
     const y = value[Node.valuePropsCoordinate.y]
     return Node.newNodeValueCoordinate({ x, y, srsId })
   },
-  [NodeDef.nodeDefType.date]: ({ value, headers }) => {
-    const val = singlePropValueConverter({ value })
-    let dateParsed = null
-    allowedDateFormats.some((format) => {
-      dateParsed = DateUtils.parse(val, format)
-      return DateUtils.isValidDateObject(dateParsed)
-    })
-    if (!DateUtils.isValidDateObject(dateParsed)) {
-      throw new SystemError('validationErrors.dataImport.invalidDate', { headers, value: val })
-    }
-    return DateUtils.formatDateISO(dateParsed)
-  },
+  [NodeDef.nodeDefType.date]: ({ value, headers }) =>
+    extractDateOrTime({
+      value,
+      allowedFormats: allowedDateFormats,
+      formatTo: DateUtils.formats.dateISO,
+      headers,
+      errorKey: 'validationErrors.dataImport.invalidDate',
+    }),
   [NodeDef.nodeDefType.decimal]: numericValueConverter,
   [NodeDef.nodeDefType.file]: ({ value, headers }) => {
     const { fileName, fileUuid: fileUuidInValue } = value
@@ -99,7 +111,14 @@ const valueConverterByNodeDefType = {
     throw new SystemError('validationErrors.dataImport.invalidTaxonCode', { value, headers })
   },
   [NodeDef.nodeDefType.text]: singlePropValueConverter,
-  [NodeDef.nodeDefType.time]: singlePropValueConverter,
+  [NodeDef.nodeDefType.time]: ({ value, headers }) =>
+    extractDateOrTime({
+      value,
+      allowedFormats: allowedTimeFormats,
+      formatTo: DateUtils.formats.timeStorage,
+      headers,
+      errorKey: 'validationErrors.dataImport.invalidTime',
+    }),
 }
 
 const checkAllHeadersAreValid =
@@ -224,7 +243,9 @@ const createReaderFromStream = ({
         const nodeDef = Survey.getNodeDefByUuid(nodeDefUuid)(survey)
         const valueConverter = valueConverterByNodeDefType[NodeDef.getType(nodeDef)]
         try {
-          const nodeValue = valueConverter({ survey, nodeDef, value: valueTemp, headers })
+          const nodeValue = Objects.isEmpty(valueTemp)
+            ? null
+            : valueConverter({ survey, nodeDef, value: valueTemp, headers })
           acc[nodeDefUuid] = nodeValue
         } catch (error) {
           errors.push(error)

webapp/components/form/DateTimeInput/useDateTimeInput.js

@@ -6,7 +6,7 @@ import * as DateUtils from '@core/dateUtils'
 export const useDateTimeInput = ({ onChange: onChangeProp, value, valueFormat }) => {
   const errorRef = useRef(false)
 
-  const dateValue = value ? moment(DateUtils.parse(value, valueFormat, false)) : null
+  const dateValue = value ? moment(DateUtils.parse(value, valueFormat, { keepTimeZone: false })) : null
 
   const applyChange = useCallback(
     (dateFormatted) => {

yarn.lock

@@ -2777,10 +2777,10 @@
     proj4 "^2.11.0"
     uuid "^10.0.0"
 
-"@openforis/arena-core@^1.0.1":
-  version "1.0.1"
-  resolved "https://npm.pkg.github.com/download/@openforis/arena-core/1.0.1/06b493d0832c6ef31482cfd39bf35b191c5e361c#06b493d0832c6ef31482cfd39bf35b191c5e361c"
-  integrity sha512-qtJahu2theB1TFpIPhg3DmGwPGUTDJYxHij/tmHnPh8Mr3oqK9PrlyZ1hSoVlJ4Y6gMcTGaF2fsT8zLwGOCv1A==
+"@openforis/arena-core@^1.0.2":
+  version "1.0.2"
+  resolved "https://npm.pkg.github.com/download/@openforis/arena-core/1.0.2/2226a763d2cbe2f30735aaa449f3f796ccf08ba2#2226a763d2cbe2f30735aaa449f3f796ccf08ba2"
+  integrity sha512-b22MuMmqPbRbKrJu/JMMAYfPyTGmbcVmiBEZ/0BB+CrWKSxQdD1wZ9k73gpCccbsHkQVcKrf37npO78ga8MCcw==
   dependencies:
     "@jsep-plugin/regex" "^1.0.3"
     bignumber.js "^9.1.2"