diff --git a/apps/jetstream/src/app/components/load-records/LoadRecords.tsx b/apps/jetstream/src/app/components/load-records/LoadRecords.tsx index 2b56697eb..696a769d9 100644 --- a/apps/jetstream/src/app/components/load-records/LoadRecords.tsx +++ b/apps/jetstream/src/app/components/load-records/LoadRecords.tsx @@ -111,10 +111,10 @@ export const LoadRecords: FunctionComponent = ({ featureFlags const resetApiModeState = useResetRecoilState(fromLoadRecordsState.apiModeState); const resetBatchSizeState = useResetRecoilState(fromLoadRecordsState.batchSizeState); - const resetBatchSizeErrorState = useResetRecoilState(fromLoadRecordsState.batchSizeErrorState); const resetInsertNullsState = useResetRecoilState(fromLoadRecordsState.insertNullsState); const resetSerialModeState = useResetRecoilState(fromLoadRecordsState.serialModeState); - const resetDateFormatState = useResetRecoilState(fromLoadRecordsState.dateFormatState); + const resetTrialRunState = useResetRecoilState(fromLoadRecordsState.trialRunState); + const resetTrialRunSizeState = useResetRecoilState(fromLoadRecordsState.trialRunSizeState); useEffect(() => { isMounted.current = true; @@ -137,17 +137,16 @@ export const LoadRecords: FunctionComponent = ({ featureFlags setBatchSize(getMaxBatchSize(apiMode)); setSerialMode(apiMode === 'BATCH'); - resetBatchSizeErrorState(); resetInsertNullsState(); - resetDateFormatState(); + resetTrialRunState(); + resetTrialRunSizeState(); }, [ inputFileData, inputZipFileData, - resetBatchSizeErrorState, resetBatchSizeState, - resetDateFormatState, + resetTrialRunSizeState, + resetTrialRunState, resetInsertNullsState, - resetSerialModeState, setApiMode, setBatchSize, setSerialMode, @@ -168,10 +167,10 @@ export const LoadRecords: FunctionComponent = ({ featureFlags resetInputZipFilename(); resetApiModeState(); resetBatchSizeState(); - resetBatchSizeErrorState(); resetInsertNullsState(); resetSerialModeState(); - resetDateFormatState(); + resetTrialRunState(); + resetTrialRunSizeState(); } }; }, [ @@ -187,10 +186,10 @@ export const LoadRecords: FunctionComponent = ({ featureFlags resetInputZipFilename, resetApiModeState, resetBatchSizeState, - resetBatchSizeErrorState, resetInsertNullsState, resetSerialModeState, - resetDateFormatState, + resetTrialRunState, + resetTrialRunSizeState, ]); useEffect(() => { @@ -382,10 +381,10 @@ export const LoadRecords: FunctionComponent = ({ featureFlags setExternalId(''); resetApiModeState(); resetBatchSizeState(); - resetBatchSizeErrorState(); resetInsertNullsState(); resetSerialModeState(); - resetDateFormatState(); + resetTrialRunState(); + resetTrialRunSizeState(); trackEvent(ANALYTICS_KEYS.load_StartOver, { page: currentStep.name }); } diff --git a/apps/jetstream/src/app/components/load-records/components/load-results/LoadRecordsResults.tsx b/apps/jetstream/src/app/components/load-records/components/load-results/LoadRecordsResults.tsx index 7b3c85241..0675d1174 100644 --- a/apps/jetstream/src/app/components/load-records/components/load-results/LoadRecordsResults.tsx +++ b/apps/jetstream/src/app/components/load-records/components/load-results/LoadRecordsResults.tsx @@ -38,7 +38,7 @@ export const LoadRecordsResults: FunctionComponent = ({ onFinish, }) => { return ( -
+
{apiMode === 'BULK' && ( (); SUPPORTED_ATTACHMENT_OBJECTS.set('Attachment', { bodyField: 'Body' }); SUPPORTED_ATTACHMENT_OBJECTS.set('Document', { bodyField: 'Body' }); SUPPORTED_ATTACHMENT_OBJECTS.set('ContentVersion', { bodyField: 'VersionData' }); +const DATE_FIELDS = new Set(['date', 'datetime']); export interface LoadSavedMappingItem { key: string; // object:createdDate @@ -169,11 +170,6 @@ export const batchSizeState = atom>({ default: MAX_BULK, }); -export const batchSizeErrorState = atom>({ - key: 'batchSizeErrorState', - default: null, -}); - export const insertNullsState = atom({ key: 'insertNullsState', default: false, @@ -184,9 +180,24 @@ export const serialModeState = atom({ default: false, }); +export const trialRunState = atom({ + key: 'trialRunState', + default: false, +}); + +export const trialRunSizeState = atom>({ + key: 'trialRunSizeState', + default: 1, +}); + export const dateFormatState = atom({ key: 'dateFormatState', - default: DATE_FORMATS.MM_DD_YYYY, + default: detectDateFormatForLocale(), +}); + +export const selectHasDateFieldMapped = selector({ + key: 'load.selectHasDateFieldMapped', + get: ({ get }) => Object.values(get(fieldMappingState)).some((item) => item.fieldMetadata && DATE_FIELDS.has(item.fieldMetadata.type)), }); export const selectBatchSizeError = selector({ @@ -218,6 +229,18 @@ export const selectBatchApiLimitError = selector({ }, }); +export const selectTrialRunSizeError = selector({ + key: 'load.selectTrialRunSizeError', + get: ({ get }) => { + const inputLength = get(inputFileDataState)?.length || 1; + const trialRunSize = get(trialRunSizeState) || 1; + if (!isNumber(trialRunSize) || trialRunSize <= 0 || trialRunSize >= inputLength) { + return `Must be between 1 and ${formatNumber(inputLength - 1)}`; + } + return null; + }, +}); + export const selectBulkApiModeLabel = selector({ key: 'load.selectBulkApiModeLabel', get: ({ get }) => { diff --git a/apps/jetstream/src/app/components/load-records/steps/PerformLoad.tsx b/apps/jetstream/src/app/components/load-records/steps/PerformLoad.tsx index d11cad41d..03db5c8a1 100644 --- a/apps/jetstream/src/app/components/load-records/steps/PerformLoad.tsx +++ b/apps/jetstream/src/app/components/load-records/steps/PerformLoad.tsx @@ -1,7 +1,8 @@ import { ANALYTICS_KEYS, DATE_FORMATS, TITLES } from '@jetstream/shared/constants'; import { formatNumber, useNonInitialEffect } from '@jetstream/shared/ui-utils'; +import { pluralizeIfMultiple } from '@jetstream/shared/utils'; import { InsertUpdateUpsertDelete, Maybe, SalesforceOrgUi, SalesforceOrgUiType } from '@jetstream/types'; -import { Badge, Checkbox, ConfirmationModalPromise, Input, Radio, RadioGroup, Select } from '@jetstream/ui'; +import { Badge, Checkbox, ConfirmationModalPromise, Grid, Input, Radio, RadioButton, RadioGroup } from '@jetstream/ui'; import startCase from 'lodash/startCase'; import { ChangeEvent, FunctionComponent, useState } from 'react'; import { useRecoilState, useRecoilValue } from 'recoil'; @@ -14,6 +15,16 @@ import { FieldMapping } from '../load-records-types'; import * as loadRecordsState from '../load-records.state'; import { getMaxBatchSize } from '../utils/load-records-utils'; +interface LoadState { + loading: boolean; + loadInProgress: boolean; + loadInProgressTrialRun: boolean; + hasLoadResultsTrialRun: boolean; + hasLoadResults: boolean; + inputFileDataTrialRun: any[]; + inputFileDataToLoad: any[]; +} + export interface LoadRecordsPerformLoadProps { selectedOrg: SalesforceOrgUi; orgType: Maybe; @@ -42,30 +53,49 @@ export const LoadRecordsPerformLoad: FunctionComponent(0); + const [loadNumberTrialRun, setLoadNumberTrialRun] = useState(0); const [apiMode, setApiMode] = useRecoilState(loadRecordsState.apiModeState); const [batchSize, setBatchSize] = useRecoilState(loadRecordsState.batchSizeState); const [insertNulls, setInsertNulls] = useRecoilState(loadRecordsState.insertNullsState); const [serialMode, setSerialMode] = useRecoilState(loadRecordsState.serialModeState); + const [trialRun, setTrialRun] = useRecoilState(loadRecordsState.trialRunState); + const [trialRunSize, setTrialRunSize] = useRecoilState(loadRecordsState.trialRunSizeState); const [dateFormat, setDateFormat] = useRecoilState(loadRecordsState.dateFormatState); + /** Only show date hint if the user has a mapped date/datetime field */ + const hasDateFieldMapped = useRecoilValue(loadRecordsState.selectHasDateFieldMapped); const batchSizeError = useRecoilValue(loadRecordsState.selectBatchSizeError); const batchApiLimitError = useRecoilValue(loadRecordsState.selectBatchApiLimitError); + const trialRunSizeError = useRecoilValue(loadRecordsState.selectTrialRunSizeError); const bulkApiModeLabel = useRecoilValue(loadRecordsState.selectBulkApiModeLabel); const batchApiModeLabel = useRecoilValue(loadRecordsState.selectBatchApiModeLabel); - const [loading, setLoading] = useState(false); - const [loadInProgress, setLoadInProgress] = useState(false); - const [hasLoadResults, setHasLoadResults] = useState(false); const loadTypeLabel = startCase(loadType.toLowerCase()); - const numRecordsImpactedLabel = formatNumber(inputFileData.length); const [assignmentRuleId, setAssignmentRuleId] = useState>(null); + const [ + { loading, loadInProgress, loadInProgressTrialRun, hasLoadResultsTrialRun, hasLoadResults, inputFileDataTrialRun, inputFileDataToLoad }, + setLoadState, + ] = useState(() => ({ + loading: false, + loadInProgress: false, + loadInProgressTrialRun: false, + hasLoadResultsTrialRun: false, + hasLoadResults: false, + inputFileDataTrialRun: inputFileData.slice(0, trialRunSize || 0), + inputFileDataToLoad: inputFileData.slice(trialRunSize || 0), + })); + + const numRecordsImpactedLabel = formatNumber(inputFileDataToLoad.length); + const numRecordsImpactedTrialRunLabel = formatNumber(inputFileDataTrialRun.length); useNonInitialEffect(() => { setBatchSize(getMaxBatchSize(apiMode)); - if (hasLoadResults) { - setHasLoadResults(false); - } + setLoadState((prevState) => ({ + ...prevState, + hasLoadResults: false, + hasLoadResultsTrialRun: false, + })); if (apiMode === 'BATCH' && !serialMode) { setSerialMode(true); } else if (apiMode === 'BULK' && serialMode) { @@ -74,6 +104,16 @@ export const LoadRecordsPerformLoad: FunctionComponent { + setLoadState((prevState) => ({ + ...prevState, + hasLoadResults: false, + hasLoadResultsTrialRun: false, + inputFileDataTrialRun: trialRun && trialRunSize ? inputFileData.slice(0, trialRunSize) : [], + inputFileDataToLoad: trialRun && trialRunSize ? inputFileData.slice(trialRunSize || 0) : inputFileData, + })); + }, [trialRun, trialRunSize, inputFileData]); + function handleBatchSize(event: ChangeEvent) { const value = Number.parseInt(event.target.value); if (Number.isInteger(value)) { @@ -83,21 +123,33 @@ export const LoadRecordsPerformLoad: FunctionComponent) { - setDateFormat(event.target.value); + function handletrialRunSize(event: ChangeEvent) { + const value = Number.parseInt(event.target.value); + if (Number.isInteger(value)) { + setTrialRunSize(value); + } else if (!event.target.value) { + setTrialRunSize(null); + } } - async function handleStartLoad() { + async function handleStartLoad(isTrialRun = false) { if ( loadNumber === 0 || (await ConfirmationModalPromise({ content: 'This file has already been loaded, are you sure you want to load it again?', })) ) { - setLoadNumber(loadNumber + 1); - setLoading(true); - setLoadInProgress(true); - setHasLoadResults(false); + if (isTrialRun) { + setLoadNumberTrialRun(loadNumberTrialRun + 1); + } else { + setLoadNumber(loadNumber + 1); + } + setLoadState((prevState) => { + if (isTrialRun) { + return { ...prevState, loading: true, loadInProgressTrialRun: true, hasLoadResultsTrialRun: false, hasLoadResults: false }; + } + return { ...prevState, loading: true, loadInProgress: true, hasLoadResults: false }; + }); onIsLoading(true); trackEvent(ANALYTICS_KEYS.load_Submitted, { loadType, @@ -106,7 +158,10 @@ export const LoadRecordsPerformLoad: FunctionComponent type === 'STATIC').length, @@ -115,16 +170,19 @@ export const LoadRecordsPerformLoad: FunctionComponent { + if (isTrialRun) { + return { ...prevState, loading: false, loadInProgressTrialRun: false, hasLoadResultsTrialRun: true }; + } + return { ...prevState, loading: false, loadInProgress: false, hasLoadResults: true }; + }); onIsLoading(false); document.title = `${formatNumber(success)} Success - ${formatNumber(failure)} Failed ${TITLES.BAR_JETSTREAM}`; } function hasDataInputError(): boolean { - return !!batchSizeError || !!batchApiLimitError; + return !!batchSizeError || !!batchApiLimitError || (trialRun && !!trialRunSizeError); } return ( @@ -207,31 +265,81 @@ export const LoadRecordsPerformLoad: FunctionComponent - - - - - - + + + + + )} + + {!inputZipFileData && ( + <> + + + {trialRun && ( + + + + )} + + )}

Summary

@@ -241,26 +349,63 @@ export const LoadRecordsPerformLoad: FunctionComponent {selectedOrg.username}
-
- -
+ + {trialRun && ( +
+ +
+ )} +
+ +
+

Results

+ {/* DRY RUN LOAD */} + {trialRun && (loadInProgressTrialRun || hasLoadResultsTrialRun) && ( + handleFinishLoad(results, true)} + /> + )} + {/* STANDARD LOAD */} {(loadInProgress || hasLoadResults) && ( handleFinishLoad(results)} /> )}
diff --git a/apps/jetstream/src/app/components/load-records/steps/SelectObjectAndFile.tsx b/apps/jetstream/src/app/components/load-records/steps/SelectObjectAndFile.tsx index f33def258..d6f1b1691 100644 --- a/apps/jetstream/src/app/components/load-records/steps/SelectObjectAndFile.tsx +++ b/apps/jetstream/src/app/components/load-records/steps/SelectObjectAndFile.tsx @@ -75,9 +75,9 @@ export const LoadRecordsSelectObjectAndFile: FunctionComponent { const hasGoogleInputConfigured = !!googleApiConfig?.apiKey && !!googleApiConfig?.appId && !!googleApiConfig?.clientId; - async function handleFile({ content, filename, isPasteFromClipboard }: InputReadFileContent) { + async function handleFile({ content, filename, isPasteFromClipboard, extension }: InputReadFileContent) { try { - const { data, headers, errors } = await parseFile(content, { onParsedMultipleWorkbooks, isPasteFromClipboard }); + const { data, headers, errors } = await parseFile(content, { onParsedMultipleWorkbooks, isPasteFromClipboard, extension }); onFileChange(data, headers, filename, 'local'); if (errors.length > 0) { logger.warn(errors); @@ -166,7 +166,7 @@ export const LoadRecordsSelectObjectAndFile: FunctionComponent Promise; isBinaryString?: boolean; isPasteFromClipboard?: boolean; + extension?: string; } ): Promise<{ data: any[]; @@ -1145,11 +1146,23 @@ export async function parseFile( options = options || {}; if (!options.isBinaryString && isString(content)) { // csv - read from papaparse - const csvResult = parseCsv(content, { - delimiter: options.isPasteFromClipboard ? undefined : detectDelimiter(), + let csvResult = parseCsv(content, { + delimiter: options.isPasteFromClipboard ? undefined : detectDelimiter(options.extension), header: true, skipEmptyLines: true, }); + // Check if it is likely an incorrect delimiter was used and re-parse file with auto-detect delimiter + if ( + Array.isArray(csvResult.meta.fields) && + csvResult.meta.fields.length === 1 && + ((csvResult.meta.fields[0].includes(',') && csvResult.meta.delimiter === ';') || + (csvResult.meta.fields[0].includes(';') && csvResult.meta.delimiter === ',')) + ) { + csvResult = parseCsv(content, { + header: true, + skipEmptyLines: true, + }); + } return { data: csvResult.data, headers: Array.from(new Set(csvResult.meta.fields)), // remove duplicates, if any @@ -1206,7 +1219,10 @@ export function generateCsv(data: any[], options: UnparseConfig = {}): string { return unparseCsv(data, options); } -function detectDelimiter(): string { +function detectDelimiter(extension?: string): string { + if (extension === INPUT_ACCEPT_FILETYPES.TSV) { + return '\t'; + } let delimiter = ','; try { // determine if delimiter is the same as the decimal symbol in current locale @@ -1241,6 +1257,27 @@ export function convertDateToLocale(dateOrIsoDateString: string | Date, options? } } +export function detectDateFormatForLocale() { + try { + const locale = navigator.language || 'en-US'; + const testDate = new Date(2021, 11, 24); // December 24, 2021 + const formattedDate = new Intl.DateTimeFormat(locale).format(testDate); + + if (formattedDate.startsWith('12')) { + return DATE_FORMATS.MM_DD_YYYY; + } else if (formattedDate.startsWith('24')) { + return DATE_FORMATS.DD_MM_YYYY; + } else if (formattedDate.startsWith('2021')) { + return DATE_FORMATS.YYYY_MM_DD; + } + } catch (ex) { + logger.warn(`[ERROR] Exception detecting date format`, ex.message); + } + + logger.warn(`[ERROR] Falling back to ${DATE_FORMATS.MM_DD_YYYY}`); + return DATE_FORMATS.MM_DD_YYYY; +} + export function convertArrayOfObjectToArrayOfArray(data: any[], headers?: string[]): any[][] { if (!data || !data.length) { return []; diff --git a/libs/shared/utils/src/lib/utils.ts b/libs/shared/utils/src/lib/utils.ts index c6d6a08fe..ea84d0ec5 100644 --- a/libs/shared/utils/src/lib/utils.ts +++ b/libs/shared/utils/src/lib/utils.ts @@ -655,9 +655,18 @@ function buildDateFromString(value: string, dateFormat: string, representation: } function getIsoFormattedTimeFromString(time: string) { - const timeFormat = ['HH:mm:ss.SSSZ', 'HH:mm:ssZ', 'p', 'pp', 'hh:mm a', 'hh:mm:ss a', 'hh:mma', 'hh:mm:ssa', 'HH:mm', 'HH:mm:ss'].find( - (format) => isMatch(time, format) - ); + const timeFormat = [ + `HH:mm:ss.SSS'Z'`, + `HH:mm:ss'Z'`, + 'p', + 'pp', + 'hh:mm a', + 'hh:mm:ss a', + 'hh:mma', + 'hh:mm:ssa', + 'HH:mm', + 'HH:mm:ss', + ].find((format) => isMatch(time, format)); const formattedTime = timeFormat ? formatISODate(parseDate(time, timeFormat, new Date()), { representation: 'time' }) : null; return formattedTime; } diff --git a/libs/types/src/lib/ui/types.ts b/libs/types/src/lib/ui/types.ts index e43997922..af42b8214 100644 --- a/libs/types/src/lib/ui/types.ts +++ b/libs/types/src/lib/ui/types.ts @@ -203,9 +203,10 @@ export type MimeTypeJson = 'application/json;charset=utf-8'; export type MimeTypeXML = 'text/xml;charset=utf-8'; export type MimeTypeGSheet = 'application/vnd.google-apps.spreadsheet'; -export type InputAcceptType = InputAcceptTypeZip | InputAcceptTypeCsv | InputAcceptTypeExcel | InputAcceptTypeXml; +export type InputAcceptType = InputAcceptTypeZip | InputAcceptTypeCsv | InputAcceptTypeTsv | InputAcceptTypeExcel | InputAcceptTypeXml; export type InputAcceptTypeZip = '.zip'; export type InputAcceptTypeCsv = '.csv'; +export type InputAcceptTypeTsv = '.tsv'; export type InputAcceptTypeExcel = '.xlsx'; export type InputAcceptTypeXml = '.xml'; diff --git a/libs/ui/src/lib/form/file-selector/FileSelector.tsx b/libs/ui/src/lib/form/file-selector/FileSelector.tsx index 17fa75a98..af415c3a8 100644 --- a/libs/ui/src/lib/form/file-selector/FileSelector.tsx +++ b/libs/ui/src/lib/form/file-selector/FileSelector.tsx @@ -149,7 +149,7 @@ export const FileSelector: FunctionComponent = ({ setManagedFilename(file.name); - const readAsArrayBuffer = extension !== '.csv' && extension !== '.xml'; + const readAsArrayBuffer = extension !== '.csv' && extension !== '.tsv' && extension !== '.xml'; const content = await (readAsArrayBuffer ? readFile(file, 'array_buffer') : readFile(file, 'text')); onReadFile({ filename: file.name, extension, content });