From 7e6f1efe304c08d45679794f0e3672155cdcef4e Mon Sep 17 00:00:00 2001 From: Ahmed Bouhuolia Date: Thu, 29 Aug 2024 19:48:58 +0200 Subject: [PATCH] feat: Add date format controll to import --- .../controllers/Import/ImportController.ts | 1 + .../src/services/Import/ImportFileCommon.ts | 12 +++-- .../Import/ImportFileDataValidator.ts | 14 +++-- packages/server/src/services/Import/_utils.ts | 51 ++++++++++++++++--- .../Import/ImportFileMapping.module.scss | 14 +++-- .../containers/Import/ImportFileMapping.tsx | 27 ++++++++-- .../Import/ImportFileMappingBoot.tsx | 15 +++++- .../containers/Import/ImportFileProvider.tsx | 1 + .../Import/ImportFileUploadStep.module.scss | 4 +- .../webapp/src/containers/Import/_types.ts | 11 +++- .../webapp/src/containers/Import/_utils.ts | 37 ++++++++------ 11 files changed, 141 insertions(+), 46 deletions(-) diff --git a/packages/server/src/api/controllers/Import/ImportController.ts b/packages/server/src/api/controllers/Import/ImportController.ts index 17b60bc952..256244412f 100644 --- a/packages/server/src/api/controllers/Import/ImportController.ts +++ b/packages/server/src/api/controllers/Import/ImportController.ts @@ -40,6 +40,7 @@ export class ImportController extends BaseController { body('mapping.*.group').optional(), body('mapping.*.from').exists(), body('mapping.*.to').exists(), + body('mapping.*.dateFormat').optional({ nullable: true }), ], this.validationResult, this.asyncMiddleware(this.mapping.bind(this)), diff --git a/packages/server/src/services/Import/ImportFileCommon.ts b/packages/server/src/services/Import/ImportFileCommon.ts index 4e9179bdda..cdcdf5e992 100644 --- a/packages/server/src/services/Import/ImportFileCommon.ts +++ b/packages/server/src/services/Import/ImportFileCommon.ts @@ -12,7 +12,11 @@ import { ImportableContext, } from './interfaces'; import { ServiceError } from '@/exceptions'; -import { getUniqueImportableValue, trimObject } from './_utils'; +import { + convertMappingsToObject, + getUniqueImportableValue, + trimObject, +} from './_utils'; import { ImportableResources } from './ImportableResources'; import ResourceService from '../Resource/ResourceService'; import { Import } from '@/system/models'; @@ -43,7 +47,6 @@ export class ImportFileCommon { return XLSX.utils.sheet_to_json(worksheet, {}); } - /** * Imports the given parsed data to the resource storage through registered importable service. * @param {number} tenantId - @@ -82,11 +85,14 @@ export class ImportFileCommon { rowNumber, uniqueValue, }; + const mappingSettings = convertMappingsToObject(importFile.columnsParsed); + try { // Validate the DTO object before passing it to the service layer. await this.importFileValidator.validateData( resourceFields, - transformedDTO + transformedDTO, + mappingSettings ); try { // Run the importable function and listen to the errors. diff --git a/packages/server/src/services/Import/ImportFileDataValidator.ts b/packages/server/src/services/Import/ImportFileDataValidator.ts index 143533c668..68fd18a685 100644 --- a/packages/server/src/services/Import/ImportFileDataValidator.ts +++ b/packages/server/src/services/Import/ImportFileDataValidator.ts @@ -1,5 +1,9 @@ import { Service } from 'typedi'; -import { ImportInsertError, ResourceMetaFieldsMap } from './interfaces'; +import { + ImportInsertError, + ImportMappingAttr, + ResourceMetaFieldsMap, +} from './interfaces'; import { ERRORS, convertFieldsToYupValidation } from './_utils'; import { IModelMeta } from '@/interfaces'; import { ServiceError } from '@/exceptions'; @@ -24,9 +28,13 @@ export class ImportFileDataValidator { */ public async validateData( importableFields: ResourceMetaFieldsMap, - data: Record + data: Record, + mappingSettings: Record = {} ): Promise { - const YupSchema = convertFieldsToYupValidation(importableFields); + const YupSchema = convertFieldsToYupValidation( + importableFields, + mappingSettings + ); const _data = { ...data }; try { diff --git a/packages/server/src/services/Import/_utils.ts b/packages/server/src/services/Import/_utils.ts index e99b8658bc..6895ab4721 100644 --- a/packages/server/src/services/Import/_utils.ts +++ b/packages/server/src/services/Import/_utils.ts @@ -17,9 +17,10 @@ import { head, split, last, + set, } from 'lodash'; import pluralize from 'pluralize'; -import { ResourceMetaFieldsMap } from './interfaces'; +import { ImportMappingAttr, ResourceMetaFieldsMap } from './interfaces'; import { IModelMetaField, IModelMetaField2 } from '@/interfaces'; import { ServiceError } from '@/exceptions'; import { multiNumberParse } from '@/utils/multi-number-parse'; @@ -58,11 +59,19 @@ export function trimObject(obj: Record) { * @param {ResourceMetaFieldsMap} fields * @returns {Yup} */ -export const convertFieldsToYupValidation = (fields: ResourceMetaFieldsMap) => { +export const convertFieldsToYupValidation = ( + fields: ResourceMetaFieldsMap, + parentFieldName: string = '', + mappingSettings: Record = {} +) => { const yupSchema = {}; Object.keys(fields).forEach((fieldName: string) => { const field = fields[fieldName] as IModelMetaField; + const fieldPath = parentFieldName + ? `${parentFieldName}.${fieldName}` + : fieldName; + let fieldSchema; fieldSchema = Yup.string().label(field.name); @@ -105,13 +114,23 @@ export const convertFieldsToYupValidation = (fields: ResourceMetaFieldsMap) => { if (!val) { return true; } - return moment(val, 'YYYY-MM-DD', true).isValid(); + const fieldDateFormat = + (get( + mappingSettings, + `${fieldPath}.dateFormat` + ) as unknown as string) || 'YYYY-MM-DD'; + + return moment(val, fieldDateFormat, true).isValid(); } ); } else if (field.fieldType === 'url') { fieldSchema = fieldSchema.url(); } else if (field.fieldType === 'collection') { - const nestedFieldShema = convertFieldsToYupValidation(field.fields); + const nestedFieldShema = convertFieldsToYupValidation( + field.fields, + field.name, + mappingSettings + ); fieldSchema = Yup.array().label(field.name); if (!isUndefined(field.collectionMaxLength)) { @@ -258,6 +277,7 @@ export const getResourceColumns = (resourceColumns: { ]) => { const extra: Record = {}; const key = fieldKey; + const type = field.fieldType; if (group) { extra.group = group; @@ -270,6 +290,7 @@ export const getResourceColumns = (resourceColumns: { name, required, hint: importHint, + type, order, ...extra, }; @@ -322,6 +343,8 @@ export const valueParser = }); const result = await relationQuery.first(); _value = get(result, 'id'); + } else if (field.fieldType === 'date') { + } else if (field.fieldType === 'collection') { const ObjectFieldKey = key.includes('.') ? key.split('.')[1] : key; const _valueParser = valueParser(fields, tenantModels); @@ -433,8 +456,8 @@ export const getMapToPath = (to: string, group = '') => group ? `${group}.${to}` : to; export const getImportsStoragePath = () => { - return path.join(global.__storage_dir, `/imports`); -} + return path.join(global.__storage_dir, `/imports`); +}; /** * Deletes the imported file from the storage and database. @@ -457,3 +480,19 @@ export const readImportFile = (filename: string) => { return fs.readFile(`${filePath}/${filename}`); }; + +/** + * Converts an array of mapping objects to a structured object. + * @param {Array} mappings - Array of mapping objects. + * @returns {Object} - Structured object based on the mappings. + */ +export const convertMappingsToObject = (mappings) => { + return mappings.reduce((acc, mapping) => { + const { to, group } = mapping; + const key = group ? `['${group}.${to}']` : to; + + set(acc, key, mapping); + + return acc; + }, {}); +}; diff --git a/packages/webapp/src/containers/Import/ImportFileMapping.module.scss b/packages/webapp/src/containers/Import/ImportFileMapping.module.scss index 156da57fb0..7210e460bc 100644 --- a/packages/webapp/src/containers/Import/ImportFileMapping.module.scss +++ b/packages/webapp/src/containers/Import/ImportFileMapping.module.scss @@ -14,7 +14,7 @@ th.label, td.label{ - width: 32% !important; + width: 30% !important; } thead{ @@ -31,16 +31,14 @@ tr td { vertical-align: middle; } - - tr td{ - :global(.bp4-popover-target .bp4-button), - :global(.bp4-popover-wrapper){ - max-width: 250px; - } - } } } .requiredSign{ color: rgb(250, 82, 82); +} + +.columnSelectButton{ + max-width: 250px; + min-width: 250px; } \ No newline at end of file diff --git a/packages/webapp/src/containers/Import/ImportFileMapping.tsx b/packages/webapp/src/containers/Import/ImportFileMapping.tsx index 7efae0340c..3dc7b28de5 100644 --- a/packages/webapp/src/containers/Import/ImportFileMapping.tsx +++ b/packages/webapp/src/containers/Import/ImportFileMapping.tsx @@ -8,9 +8,12 @@ import { EntityColumnField, useImportFileContext } from './ImportFileProvider'; import { CLASSES } from '@/constants'; import { ImportFileContainer } from './ImportFileContainer'; import { ImportStepperStep } from './_types'; -import { ImportFileMapBootProvider } from './ImportFileMappingBoot'; +import { + ImportFileMapBootProvider, + useImportFileMapBootContext, +} from './ImportFileMappingBoot'; import styles from './ImportFileMapping.module.scss'; -import { getFieldKey } from './_utils'; +import { getDateFieldKey, getFieldKey } from './_utils'; export function ImportFileMapping() { const { importId, entityColumns } = useImportFileContext(); @@ -82,6 +85,7 @@ interface ImportFileMappingFieldsProps { */ function ImportFileMappingFields({ fields }: ImportFileMappingFieldsProps) { const { sheetColumns } = useImportFileContext(); + const { dateFormats } = useImportFileMapBootContext(); const items = useMemo( () => sheetColumns.map((column) => ({ value: column, text: column })), @@ -95,22 +99,35 @@ function ImportFileMappingFields({ fields }: ImportFileMappingFieldsProps) { {column.required && *} - + {column.hint && ( )} + {column.type === 'date' && ( + + )} ), - [items], + [items, dateFormats], ); const columns = useMemo( () => fields.map(columnMapper), diff --git a/packages/webapp/src/containers/Import/ImportFileMappingBoot.tsx b/packages/webapp/src/containers/Import/ImportFileMappingBoot.tsx index fb0b7f496b..cdb8888fea 100644 --- a/packages/webapp/src/containers/Import/ImportFileMappingBoot.tsx +++ b/packages/webapp/src/containers/Import/ImportFileMappingBoot.tsx @@ -2,8 +2,11 @@ import { Spinner } from '@blueprintjs/core'; import React, { createContext, useContext } from 'react'; import { Box } from '@/components'; import { useImportFileMeta } from '@/hooks/query/import'; +import { useDateFormats } from '@/hooks/query'; -interface ImportFileMapBootContextValue {} +interface ImportFileMapBootContextValue { + dateFormats: Array; +} const ImportFileMapBootContext = createContext( {} as ImportFileMapBootContextValue, @@ -39,14 +42,22 @@ export const ImportFileMapBootProvider = ({ enabled: Boolean(importId), }); + // Fetch date format options. + const { data: dateFormats, isLoading: isDateFormatsLoading } = + useDateFormats(); + const value = { importFile, isImportFileLoading, isImportFileFetching, + dateFormats, + isDateFormatsLoading, }; + const isLoading = isDateFormatsLoading || isImportFileLoading; + return ( - {isImportFileLoading ? ( + {isLoading ? ( diff --git a/packages/webapp/src/containers/Import/ImportFileProvider.tsx b/packages/webapp/src/containers/Import/ImportFileProvider.tsx index 0e41410b55..5a7eb85cb8 100644 --- a/packages/webapp/src/containers/Import/ImportFileProvider.tsx +++ b/packages/webapp/src/containers/Import/ImportFileProvider.tsx @@ -13,6 +13,7 @@ export type EntityColumnField = { required?: boolean; hint?: string; group?: string; + type: string; }; export interface EntityColumn { diff --git a/packages/webapp/src/containers/Import/ImportFileUploadStep.module.scss b/packages/webapp/src/containers/Import/ImportFileUploadStep.module.scss index 95bfd50fd3..cc79038415 100644 --- a/packages/webapp/src/containers/Import/ImportFileUploadStep.module.scss +++ b/packages/webapp/src/containers/Import/ImportFileUploadStep.module.scss @@ -6,8 +6,8 @@ flex: 1; padding: 32px 20px; padding-bottom: 80px; - min-width: 660px; - max-width: 760px; + min-width: 800px; + max-width: 800px; width: 75%; margin-left: auto; margin-right: auto; diff --git a/packages/webapp/src/containers/Import/_types.ts b/packages/webapp/src/containers/Import/_types.ts index a5f2ca79f9..dfa55ad9ed 100644 --- a/packages/webapp/src/containers/Import/_types.ts +++ b/packages/webapp/src/containers/Import/_types.ts @@ -12,4 +12,13 @@ export interface ImportFileMappingFormProps { children: React.ReactNode; } -export type ImportFileMappingFormValues = Record; +export type ImportFileMappingFormValues = Record< + string, + { from: string | null; dateFormat?: string } +>; + +export type ImportFileMappingRes = { + from: string; + to: string; + group: string; +}[]; diff --git a/packages/webapp/src/containers/Import/_utils.ts b/packages/webapp/src/containers/Import/_utils.ts index fdd62f0cf8..d81e2cd1d1 100644 --- a/packages/webapp/src/containers/Import/_utils.ts +++ b/packages/webapp/src/containers/Import/_utils.ts @@ -17,13 +17,15 @@ import { } from './ImportFileProvider'; import { useImportFileMapBootContext } from './ImportFileMappingBoot'; import { deepdash, transformToForm } from '@/utils'; -import { ImportFileMappingFormValues } from './_types'; +import { ImportFileMappingFormValues, ImportFileMappingRes } from './_types'; export const getFieldKey = (key: string, group = '') => { return group ? `${group}.${key}` : key; }; -type ImportFileMappingRes = { from: string; to: string; group: string }[]; +export const getDateFieldKey = (key: string, group: string = '') => { + return `${getFieldKey(key, group)}.dateFormat`; +}; /** * Transformes the mapping form values to request. @@ -34,10 +36,10 @@ export const transformValueToReq = ( value: ImportFileMappingFormValues, ): { mapping: ImportFileMappingRes[] } => { const mapping = chain(value) - .thru(deepdash.index) - .pickBy((_value, key) => !isEmpty(get(value, key))) - .map((from, key) => ({ - from, + .pickBy((_value, key) => !isEmpty(_value) && _value?.from) + .map((_value, key) => ({ + from: _value.from, + dateFormat: _value.dateFormat, to: key.includes('.') ? last(key.split('.')) : key, group: key.includes('.') ? head(key.split('.')) : '', })) @@ -52,19 +54,23 @@ export const transformValueToReq = ( * @returns {Record} */ export const transformResToFormValues = ( - value: { from: string; to: string , group: string }[], + value: { from: string; to: string; group: string }[], ): Record => { return value?.reduce((acc, map) => { - const path = map?.group ? `${map.group}.${map.to}` : map.to; - set(acc, path, map.from); + const path = map?.group ? `['${map.group}.${map.to}']` : map.to; + const dateFormatObj = map?.dateFormat + ? { dateFormat: map?.dateFormat } + : {}; + + set(acc, path, { from: map?.from, ...dateFormatObj }); return acc; }, {}); }; /** - * Retrieves the initial values of mapping form. - * @param {EntityColumn[]} entityColumns - * @param {SheetColumn[]} sheetColumns + * Retrieves the initial values of mapping form. + * @param {EntityColumn[]} entityColumns + * @param {SheetColumn[]} sheetColumns */ const getInitialDefaultValues = ( entityColumns: EntityColumn[], @@ -76,10 +82,10 @@ const getInitialDefaultValues = ( const _matched = sheetColumns.find( (column) => lowerCase(column) === _name, ); - const _key = groupKey ? `${groupKey}.${key}` : key; - const _value = _matched ? _matched : ''; + const path = groupKey ? `['${groupKey}.${key}']` : key; + const from = _matched ? _matched : ''; - set(acc, _key, _value); + set(acc, path, { from }); }); return acc; }, {}); @@ -102,7 +108,6 @@ export const useImportFileMappingInitialValues = () => { () => getInitialDefaultValues(entityColumns, sheetColumns), [entityColumns, sheetColumns], ); - return useMemo>( () => assign(