From 1c304c4e92351d730213a24ac14d4454168df156 Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Mon, 2 Dec 2024 17:34:52 +0800 Subject: [PATCH 01/35] feat: add columns for mrf responses page --- .../ResponsesTable/ResponsesTable.tsx | 53 +++++++++++++++++-- shared/types/submission.ts | 8 ++- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx b/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx index 2fded6fec6..a660e86f54 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx @@ -20,7 +20,7 @@ import { getNetAmount } from './utils' type ResponseColumnData = SubmissionMetadata -const RESPONSE_TABLE_COLUMNS: Column[] = [ +const NON_MRF_RESPONSE_TABLE_COLUMNS: Column[] = [ { Header: '#', accessor: 'number', @@ -43,6 +43,45 @@ const RESPONSE_TABLE_COLUMNS: Column[] = [ disableResizing: true, }, ] + +const MRF_RESPONSE_TABLE_COLUMNS: Column[] = [ + { + Header: '#', + accessor: 'number', + minWidth: 30, + width: 40, + maxWidth: 50, + }, + { + Header: 'Response ID', + accessor: 'refNo', + minWidth: 100, + maxWidth: 150, + width: 120, + }, + { + Header: 'Status', + accessor: 'workflowStatus', + minWidth: 50, + maxWidth: 100, + width: 50, + }, + { + Header: 'Current Step', + accessor: 'workflowStep', + minWidth: 50, + maxWidth: 100, + width: 50, + }, + { + Header: 'Timestamp', + accessor: 'submissionTime', + minWidth: 250, + width: 250, + disableResizing: true, + }, +] + const PAYMENT_COLUMNS: Column[] = [ { Header: 'Email', @@ -106,7 +145,7 @@ const PAYMENT_COLUMNS: Column[] = [ ] const PAYMENT_RESPONSE_TABLE_COLUMNS = - RESPONSE_TABLE_COLUMNS.concat(PAYMENT_COLUMNS) + NON_MRF_RESPONSE_TABLE_COLUMNS.concat(PAYMENT_COLUMNS) export const ResponsesTable = () => { const { data: form } = useAdminForm() @@ -114,6 +153,8 @@ export const ResponsesTable = () => { form?.responseMode === FormResponseMode.Encrypt ? form.payments_field.enabled : false + const isMultiRespondentForm = + form?.responseMode === FormResponseMode.Multirespondent const { currentPage: currentPage1Indexed, @@ -147,9 +188,11 @@ export const ResponsesTable = () => { gotoPage, } = useTable( { - columns: isPaymentsForm - ? PAYMENT_RESPONSE_TABLE_COLUMNS - : RESPONSE_TABLE_COLUMNS, + columns: isMultiRespondentForm + ? MRF_RESPONSE_TABLE_COLUMNS + : isPaymentsForm + ? PAYMENT_RESPONSE_TABLE_COLUMNS + : NON_MRF_RESPONSE_TABLE_COLUMNS, data: metadataToUse, // Server side pagination. manualPagination: true, diff --git a/shared/types/submission.ts b/shared/types/submission.ts index aca2f5a74c..47415fe2e5 100644 --- a/shared/types/submission.ts +++ b/shared/types/submission.ts @@ -213,13 +213,19 @@ export type SubmissionPaymentMetadata = { email: string } | null +type SubmissionMrfMetadata = { + workflow: FormWorkflowDto + workflowStep: number + workflowStatus: string | undefined // `undefined` is due to submissions before this PR not storing this value +} + export type SubmissionMetadata = { number: number refNo: SubmissionId /** Not a DateString, format is `Do MMM YYYY, h:mm:ss a` */ submissionTime: string payments: SubmissionPaymentMetadata -} +} & SubmissionMrfMetadata export type SubmissionMetadataList = { metadata: SubmissionMetadata[] From 9f89d284acebdd470cb86d65b275c4972bdb5bac Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Mon, 9 Dec 2024 22:50:37 +0800 Subject: [PATCH 02/35] feat: fix default paddings for table --- .../ResponsesTable/ResponsesTable.tsx | 98 ++++++++++--------- 1 file changed, 52 insertions(+), 46 deletions(-) diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx b/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx index a660e86f54..3123c74f78 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx @@ -24,54 +24,16 @@ const NON_MRF_RESPONSE_TABLE_COLUMNS: Column[] = [ { Header: '#', accessor: 'number', - minWidth: 80, // minWidth is only used as a limit for resizing width: 80, // width is used for both the flex-basis and flex-grow + minWidth: 80, // minWidth is only used as a limit for resizing maxWidth: 100, // maxWidth is only used as a limit for resizing }, { Header: 'Response ID', accessor: 'refNo', - minWidth: 300, width: 300, - maxWidth: 240, // maxWidth is only used as a limit for resizing - }, - { - Header: 'Timestamp', - accessor: 'submissionTime', - minWidth: 250, - width: 250, - disableResizing: true, - }, -] - -const MRF_RESPONSE_TABLE_COLUMNS: Column[] = [ - { - Header: '#', - accessor: 'number', - minWidth: 30, - width: 40, - maxWidth: 50, - }, - { - Header: 'Response ID', - accessor: 'refNo', - minWidth: 100, - maxWidth: 150, - width: 120, - }, - { - Header: 'Status', - accessor: 'workflowStatus', - minWidth: 50, - maxWidth: 100, - width: 50, - }, - { - Header: 'Current Step', - accessor: 'workflowStep', - minWidth: 50, - maxWidth: 100, - width: 50, + minWidth: 300, + maxWidth: 300, }, { Header: 'Timestamp', @@ -144,6 +106,44 @@ const PAYMENT_COLUMNS: Column[] = [ }, ] +const MRF_RESPONSE_TABLE_COLUMNS: Column[] = [ + { + Header: '#', + accessor: 'number', + width: 80, + minWidth: 80, + maxWidth: 100, + }, + { + Header: 'Response ID', + accessor: 'refNo', + width: 300, + minWidth: 300, + maxWidth: 300, + }, + { + Header: 'Status', + accessor: 'workflowStatus', + width: 176, + minWidth: 176, + maxWidth: 176, + }, + { + Header: 'Current Step', + accessor: 'workflowStep', + width: 176, + minWidth: 176, + maxWidth: 176, + }, + { + Header: 'Timestamp of first response', + accessor: 'submissionTime', + width: 320, + minWidth: 320, + maxWidth: 320, + }, +] + const PAYMENT_RESPONSE_TABLE_COLUMNS = NON_MRF_RESPONSE_TABLE_COLUMNS.concat(PAYMENT_COLUMNS) @@ -179,6 +179,16 @@ export const ResponsesTable = () => { } }, [filteredMetadata, metadata, submissionId]) + const columns = useMemo(() => { + if (isMultiRespondentForm) { + return MRF_RESPONSE_TABLE_COLUMNS + } + if (isPaymentsForm) { + return PAYMENT_RESPONSE_TABLE_COLUMNS + } + return NON_MRF_RESPONSE_TABLE_COLUMNS + }, [isMultiRespondentForm, isPaymentsForm]) + const { prepareRow, getTableProps, @@ -188,11 +198,7 @@ export const ResponsesTable = () => { gotoPage, } = useTable( { - columns: isMultiRespondentForm - ? MRF_RESPONSE_TABLE_COLUMNS - : isPaymentsForm - ? PAYMENT_RESPONSE_TABLE_COLUMNS - : NON_MRF_RESPONSE_TABLE_COLUMNS, + columns, data: metadataToUse, // Server side pagination. manualPagination: true, From 42ae99a1dfcb6b5b8150a5156e66f4546c7476eb Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Mon, 9 Dec 2024 23:49:12 +0800 Subject: [PATCH 03/35] chore: update comment --- src/app/modules/submission/submission.controller.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/modules/submission/submission.controller.ts b/src/app/modules/submission/submission.controller.ts index 9575926099..04e9ef2af3 100644 --- a/src/app/modules/submission/submission.controller.ts +++ b/src/app/modules/submission/submission.controller.ts @@ -94,7 +94,7 @@ export const getMetadata: ControllerHandler< level: PermissionLevel.Read, }), ) - // Step 3: Check whether form is encrypt mode. + // Step 3: Check whether form is encrypt or multirespondent mode. .andThen(checkFormIsEncryptModeOrMultirespondent) // Step 4: Retrieve submission metadata. .andThen((form) => { From c5c5cdc1c7760c8f961ce8f9f8df0d9f79984b8c Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Mon, 9 Dec 2024 23:51:27 +0800 Subject: [PATCH 04/35] feat: pipe step number to FE response table --- .../ResponsesTable/ResponsesTable.tsx | 14 +++- shared/types/submission.ts | 18 +++-- src/app/models/submission.server.model.ts | 74 +++++++++++++------ 3 files changed, 77 insertions(+), 29 deletions(-) diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx b/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx index 3123c74f78..bcf75ab9ea 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx @@ -123,14 +123,24 @@ const MRF_RESPONSE_TABLE_COLUMNS: Column[] = [ }, { Header: 'Status', - accessor: 'workflowStatus', + accessor: ({ mrf }) => { + if (!mrf?.workflowStatus) { + return '' + } + return mrf.workflowStatus + }, width: 176, minWidth: 176, maxWidth: 176, }, { Header: 'Current Step', - accessor: 'workflowStep', + accessor: ({ mrf }) => { + if (!(mrf?.workflowCurrentStepNumber && mrf?.workflowNumTotalSteps)) { + return '' + } + return `Step ${mrf.workflowCurrentStepNumber} of ${mrf.workflowNumTotalSteps}` + }, width: 176, minWidth: 176, maxWidth: 176, diff --git a/shared/types/submission.ts b/shared/types/submission.ts index 47415fe2e5..e634c33dff 100644 --- a/shared/types/submission.ts +++ b/shared/types/submission.ts @@ -213,19 +213,27 @@ export type SubmissionPaymentMetadata = { email: string } | null -type SubmissionMrfMetadata = { - workflow: FormWorkflowDto - workflowStep: number - workflowStatus: string | undefined // `undefined` is due to submissions before this PR not storing this value +export enum WorkflowStatus { + APPROVED = 'APPROVED', + REJECTED = 'REJECTED', + PENDING = 'PENDING', + COMPLETED = 'COMPLETED', } +export type SubmissionMrfMetadata = { + workflowCurrentStepNumber: number + workflowNumTotalSteps: number + workflowStatus: WorkflowStatus | undefined // `undefined` is due to submissions before this PR not storing this value +} | null + export type SubmissionMetadata = { number: number refNo: SubmissionId /** Not a DateString, format is `Do MMM YYYY, h:mm:ss a` */ submissionTime: string payments: SubmissionPaymentMetadata -} & SubmissionMrfMetadata + mrf: SubmissionMrfMetadata +} export type SubmissionMetadataList = { metadata: SubmissionMetadata[] diff --git a/src/app/models/submission.server.model.ts b/src/app/models/submission.server.model.ts index b7eb22459d..af1df9780d 100644 --- a/src/app/models/submission.server.model.ts +++ b/src/app/models/submission.server.model.ts @@ -12,6 +12,7 @@ import { SubmissionMetadata, SubmissionType, WebhookResponse, + WorkflowStatus, } from '../../../shared/types' import { FindFormsWithSubsAboveResult, @@ -551,20 +552,29 @@ export const MultirespondentSubmissionSchema = new Schema< }, }) +type MultiRespondentAggregates = Pick< + IMultirespondentSubmissionSchema, + 'workflowStep' | 'workflow' +> +type MultiRespondentAggregateResult = MetadataAggregateResult & + MultiRespondentAggregates + MultirespondentSubmissionSchema.statics.findSingleMetadata = function ( formId: string, submissionId: string, ): Promise { - const pageResults: Promise = this.aggregate([ - { - $match: { - submissionType: SubmissionType.Multirespondent, - form: new mongoose.Types.ObjectId(formId), - _id: new mongoose.Types.ObjectId(submissionId), + const pageResults: Promise = this.aggregate( + [ + { + $match: { + submissionType: SubmissionType.Multirespondent, + form: new mongoose.Types.ObjectId(formId), + _id: new mongoose.Types.ObjectId(submissionId), + }, }, - }, - { $limit: 1 }, - ]).exec() + { $limit: 1 }, + ], + ).exec() return Promise.resolve(pageResults).then((results) => { if (!results || results.length <= 0) { @@ -572,9 +582,12 @@ MultirespondentSubmissionSchema.statics.findSingleMetadata = function ( } const result = results[0] - + const mrfMeta = { + workflowStep: result.workflowStep, + workflow: result.workflow, + } // Build submissionMetadata object. - const metadata = buildSubmissionMetadata(result, 1) + const metadata = buildSubmissionMetadata(result, 1, undefined, mrfMeta) return metadata }) @@ -595,18 +608,22 @@ MultirespondentSubmissionSchema.statics.findAllMetadataByFormId = function ( }> { const numToSkip = (page - 1) * pageSize // return documents within the page - const pageResults: Promise = this.aggregate([ - { $match: { form: new mongoose.Types.ObjectId(formId) } }, - { $sort: { created: -1 } }, - { $skip: numToSkip }, - { $limit: pageSize }, - { - $project: { - _id: 1, - created: 1, + const pageResults: Promise = this.aggregate( + [ + { $match: { form: new mongoose.Types.ObjectId(formId) } }, + { $sort: { created: -1 } }, + { $skip: numToSkip }, + { $limit: pageSize }, + { + $project: { + _id: 1, + created: 1, + workflowStep: 1, + workflow: 1, + }, }, - }, - ]).exec() + ], + ).exec() const count = this.countDocuments({ @@ -619,10 +636,15 @@ MultirespondentSubmissionSchema.statics.findAllMetadataByFormId = function ( const metadata = results.map((result) => { const paymentMeta = result.payments?.[0] + const mrfMeta = { + workflowStep: result.workflowStep, + workflow: result.workflow, + } const metadataEntry = buildSubmissionMetadata( result, currentNumber, paymentMeta, + mrfMeta, ) currentNumber-- @@ -759,6 +781,7 @@ const buildSubmissionMetadata = ( result: MetadataAggregateResult, currentNumber: number, paymentMeta?: PaymentAggregates, + mrfMeta?: MultiRespondentAggregates, ): SubmissionMetadata => { return { number: currentNumber, @@ -779,6 +802,13 @@ const buildSubmissionMetadata = ( email: paymentMeta.email, } : null, + mrf: mrfMeta + ? { + workflowCurrentStepNumber: mrfMeta.workflowStep + 1 ?? 0, + workflowNumTotalSteps: mrfMeta.workflow?.length ?? 0, + workflowStatus: WorkflowStatus.PENDING, + } + : null, } } From b817a59bfcc0866104cd9be545426e9797e05774 Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Tue, 10 Dec 2024 01:08:22 +0800 Subject: [PATCH 05/35] feat: log step meta and pipe to fe response table --- shared/types/submission.ts | 41 ++++++++++++++--- src/app/models/submission.server.model.ts | 44 +++++++++++++++++-- .../multirespondent-submission.controller.ts | 1 - .../multirespondent-submission.service.ts | 36 ++++++++++++++- .../multirespondent-submission.types.ts | 2 + 5 files changed, 112 insertions(+), 12 deletions(-) diff --git a/shared/types/submission.ts b/shared/types/submission.ts index e634c33dff..361101764f 100644 --- a/shared/types/submission.ts +++ b/shared/types/submission.ts @@ -80,6 +80,39 @@ export type StorageModeSubmissionBase = z.infer< * Multirespondent submission typings as stored in the database. */ +export enum WorkflowStatus { + APPROVED = 'APPROVED', + REJECTED = 'REJECTED', + PENDING = 'PENDING', + COMPLETED = 'COMPLETED', +} + +export const ApprovalStatus = z.enum([ + WorkflowStatus.APPROVED, + WorkflowStatus.REJECTED, +]) + +const SubmittedNonApprovalStep = z.object({ + isApproval: z.literal(false), + submittedAt: z.string(), +}) + +export type SubmittedNonApprovalStep = z.infer + +const SubmittedApprovalStep = SubmittedNonApprovalStep.extend({ + isApproval: z.literal(true), + status: ApprovalStatus, +}) + +export type SubmittedApprovalStep = z.infer + +export const SubmittedStep = z.discriminatedUnion('isApproval', [ + SubmittedApprovalStep, + SubmittedNonApprovalStep, +]) + +export type SubmittedStep = z.infer + export const MultirespondentSubmissionBase = SubmissionBase.extend({ // Store the form fields and logic here, to use as reference for future // submitters. Don't bother to validate since this is injected by the backend. @@ -95,6 +128,7 @@ export const MultirespondentSubmissionBase = SubmissionBase.extend({ version: z.number(), workflowStep: z.number(), mrfVersion: z.number().optional(), + submittedSteps: z.array(SubmittedStep).optional(), }) export type MultirespondentSubmissionBase = z.infer< @@ -213,13 +247,6 @@ export type SubmissionPaymentMetadata = { email: string } | null -export enum WorkflowStatus { - APPROVED = 'APPROVED', - REJECTED = 'REJECTED', - PENDING = 'PENDING', - COMPLETED = 'COMPLETED', -} - export type SubmissionMrfMetadata = { workflowCurrentStepNumber: number workflowNumTotalSteps: number diff --git a/src/app/models/submission.server.model.ts b/src/app/models/submission.server.model.ts index af1df9780d..df50f1a64f 100644 --- a/src/app/models/submission.server.model.ts +++ b/src/app/models/submission.server.model.ts @@ -11,6 +11,7 @@ import { MyInfoAttribute, SubmissionMetadata, SubmissionType, + SubmittedStep, WebhookResponse, WorkflowStatus, } from '../../../shared/types' @@ -550,11 +551,15 @@ export const MultirespondentSubmissionSchema = new Schema< mrfVersion: { type: Number, }, + submittedSteps: { + type: Array, + default: [], + }, }) type MultiRespondentAggregates = Pick< IMultirespondentSubmissionSchema, - 'workflowStep' | 'workflow' + 'workflowStep' | 'workflow' | 'submittedSteps' > type MultiRespondentAggregateResult = MetadataAggregateResult & MultiRespondentAggregates @@ -585,6 +590,7 @@ MultirespondentSubmissionSchema.statics.findSingleMetadata = function ( const mrfMeta = { workflowStep: result.workflowStep, workflow: result.workflow, + submittedSteps: result.submittedSteps, } // Build submissionMetadata object. const metadata = buildSubmissionMetadata(result, 1, undefined, mrfMeta) @@ -620,6 +626,7 @@ MultirespondentSubmissionSchema.statics.findAllMetadataByFormId = function ( created: 1, workflowStep: 1, workflow: 1, + submittedSteps: 1, }, }, ], @@ -639,6 +646,7 @@ MultirespondentSubmissionSchema.statics.findAllMetadataByFormId = function ( const mrfMeta = { workflowStep: result.workflowStep, workflow: result.workflow, + submittedSteps: result.submittedSteps, } const metadataEntry = buildSubmissionMetadata( result, @@ -777,6 +785,33 @@ export const getMultirespondentSubmissionModel = ( >(SubmissionType.Multirespondent) } +const getWorkflowStatus = ( + submittedSteps: SubmittedStep[], + numTotalSteps: number, +): WorkflowStatus | undefined => { + if (submittedSteps.length <= 0 || numTotalSteps <= 0) { + // NOTE: this occurs when no steps are recorded for submissions prior to this change or when no workflow is defined. + return undefined + } + const latestSubmittedStep = submittedSteps[submittedSteps.length - 1] + if ( + latestSubmittedStep.isApproval && + latestSubmittedStep.status === WorkflowStatus.REJECTED + ) { + return WorkflowStatus.REJECTED + } + if (submittedSteps.length === numTotalSteps) { + if ( + latestSubmittedStep.isApproval && + latestSubmittedStep.status === WorkflowStatus.APPROVED + ) { + return WorkflowStatus.APPROVED + } + return WorkflowStatus.COMPLETED + } + return WorkflowStatus.PENDING +} + const buildSubmissionMetadata = ( result: MetadataAggregateResult, currentNumber: number, @@ -804,9 +839,12 @@ const buildSubmissionMetadata = ( : null, mrf: mrfMeta ? { - workflowCurrentStepNumber: mrfMeta.workflowStep + 1 ?? 0, + workflowCurrentStepNumber: mrfMeta.workflowStep + 1 ?? 0, // need to add 1 as workflowStep is 0-indexed workflowNumTotalSteps: mrfMeta.workflow?.length ?? 0, - workflowStatus: WorkflowStatus.PENDING, + workflowStatus: getWorkflowStatus( + mrfMeta.submittedSteps ?? [], + mrfMeta.workflow?.length ?? 0, + ), } : null, } diff --git a/src/app/modules/submission/multirespondent-submission/multirespondent-submission.controller.ts b/src/app/modules/submission/multirespondent-submission/multirespondent-submission.controller.ts index b237bd4672..b509db3356 100644 --- a/src/app/modules/submission/multirespondent-submission/multirespondent-submission.controller.ts +++ b/src/app/modules/submission/multirespondent-submission/multirespondent-submission.controller.ts @@ -183,7 +183,6 @@ const updateMultirespondentSubmission = async ( const updateMultiRespondentFormSubmissionResult = await updateMultiRespondentFormSubmission({ - formId, submissionId, form, encryptedPayload, diff --git a/src/app/modules/submission/multirespondent-submission/multirespondent-submission.service.ts b/src/app/modules/submission/multirespondent-submission/multirespondent-submission.service.ts index 81e3977fca..f75c4fe65c 100644 --- a/src/app/modules/submission/multirespondent-submission/multirespondent-submission.service.ts +++ b/src/app/modules/submission/multirespondent-submission/multirespondent-submission.service.ts @@ -7,6 +7,9 @@ import { FieldResponsesV3, FormResponseMode, FormWorkflowStepDto, + SubmittedApprovalStep, + SubmittedNonApprovalStep, + WorkflowStatus, } from '../../../../../shared/types' import { getMultirespondentSubmissionEditPath } from '../../../../../shared/utils/urls' import { @@ -350,6 +353,12 @@ export const createMultiRespondentFormSubmission = ({ mrfVersion, } = encryptedPayload + // For non-approval steps, we only need isApproval: false and submittedAt + const submittedStepMeta: SubmittedNonApprovalStep = { + isApproval: false, // first step cannot be approval step + submittedAt: new Date().toISOString(), + } + const submissionContent: MultirespondentSubmissionContent = { form: form._id, authType: form.authType, @@ -364,6 +373,7 @@ export const createMultiRespondentFormSubmission = ({ version, workflowStep: 0, mrfVersion, + submittedSteps: [submittedStepMeta], } const submission = new MultirespondentSubmission(submissionContent) @@ -469,7 +479,6 @@ export const updateMultiRespondentFormSubmission = ({ encryptedPayload, logMeta, }: { - formId: string submissionId: string form: IPopulatedMultirespondentForm encryptedPayload: MultirespondentSubmissionDto @@ -512,6 +521,31 @@ export const updateMultiRespondentFormSubmission = ({ mrfVersion, } = encryptedPayload + const isApprovalForm = checkIsFormApproval(form) + const isStepRejected = checkIsStepRejected({ + zeroIndexedStepNumber: workflowStep, + form, + responses: encryptedPayload.responses, + }) + const submittedStepMeta = isApprovalForm + ? ({ + status: isStepRejected + ? WorkflowStatus.REJECTED + : WorkflowStatus.APPROVED, + stepNumber: workflowStep, + isApproval: true, + submittedAt: new Date().toISOString(), + } as SubmittedApprovalStep) + : ({ + isApproval: false, + stepNumber: workflowStep, + submittedAt: new Date().toISOString(), + } as SubmittedNonApprovalStep) + + submission.submittedSteps = [ + ...(submission.submittedSteps ?? []), + submittedStepMeta, + ] submission.responseMetadata = responseMetadata submission.submissionPublicKey = submissionPublicKey submission.encryptedSubmissionSecretKey = encryptedSubmissionSecretKey diff --git a/src/app/modules/submission/multirespondent-submission/multirespondent-submission.types.ts b/src/app/modules/submission/multirespondent-submission/multirespondent-submission.types.ts index 6ba339a9ee..c27dca02ab 100644 --- a/src/app/modules/submission/multirespondent-submission/multirespondent-submission.types.ts +++ b/src/app/modules/submission/multirespondent-submission/multirespondent-submission.types.ts @@ -5,6 +5,7 @@ import { MyInfoAttribute, SubmissionErrorDto, SubmissionResponseDto, + SubmittedStep, } from '../../../../../shared/types' import { MultirespondentFormCompleteDto, @@ -88,6 +89,7 @@ export type MultirespondentSubmissionContent = { version: number workflowStep: number mrfVersion: number + submittedSteps: SubmittedStep[] } export type StrippedAttachmentResponseV3 = AttachmentResponseV3 & { From f213752d75dd7ce1d4601d08f24fb4c4c4f17dcb Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Tue, 10 Dec 2024 01:16:08 +0800 Subject: [PATCH 06/35] fix: checking is form rejected error --- .../multirespondent-submission.service.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/app/modules/submission/multirespondent-submission/multirespondent-submission.service.ts b/src/app/modules/submission/multirespondent-submission/multirespondent-submission.service.ts index f75c4fe65c..5d2f9b2080 100644 --- a/src/app/modules/submission/multirespondent-submission/multirespondent-submission.service.ts +++ b/src/app/modules/submission/multirespondent-submission/multirespondent-submission.service.ts @@ -522,11 +522,21 @@ export const updateMultiRespondentFormSubmission = ({ } = encryptedPayload const isApprovalForm = checkIsFormApproval(form) - const isStepRejected = checkIsStepRejected({ + const isStepRejectedResult = checkIsStepRejected({ zeroIndexedStepNumber: workflowStep, form, responses: encryptedPayload.responses, }) + if (isStepRejectedResult.isErr()) { + logger.error({ + message: 'Error occurred when checking if step is rejected', + meta: logMeta, + error: isStepRejectedResult.error, + }) + return errAsync(isStepRejectedResult.error) + } + + const isStepRejected = isStepRejectedResult.value const submittedStepMeta = isApprovalForm ? ({ status: isStepRejected From 6983771180853632075f15ff4846c770fe05d8a1 Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Tue, 10 Dec 2024 01:35:44 +0800 Subject: [PATCH 07/35] feat: add status badges --- .../ResponsesTable/ResponsesTable.tsx | 50 +++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx b/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx index bcf75ab9ea..aa734f36fa 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx @@ -1,4 +1,5 @@ import { useCallback, useEffect, useMemo } from 'react' +import { BiCheckDouble, BiSolidHourglass } from 'react-icons/bi' import { useNavigate } from 'react-router-dom' import { Column, @@ -7,11 +8,27 @@ import { useResizeColumns, useTable, } from 'react-table' -import { Flex, Table, Tbody, Td, Th, Thead, Tr } from '@chakra-ui/react' +import { + Flex, + Icon, + Table, + Tbody, + Td, + Text, + Th, + Thead, + Tr, +} from '@chakra-ui/react' -import { FormResponseMode, SubmissionMetadata } from '~shared/types' +import { + FormResponseMode, + SubmissionMetadata, + WorkflowStatus, +} from '~shared/types' import { centsToDollars } from '~shared/utils/payments' +import Badge from '~components/Badge' + import { useAdminForm } from '~features/admin-form/common/queries' import { useUnlockedResponses } from '../UnlockedResponsesProvider' @@ -20,6 +37,30 @@ import { getNetAmount } from './utils' type ResponseColumnData = SubmissionMetadata +const PendingBadge = () => ( + + + Pending + +) + +const CompletedBadge = () => ( + + + Completed + +) + const NON_MRF_RESPONSE_TABLE_COLUMNS: Column[] = [ { Header: '#', @@ -127,7 +168,10 @@ const MRF_RESPONSE_TABLE_COLUMNS: Column[] = [ if (!mrf?.workflowStatus) { return '' } - return mrf.workflowStatus + if (mrf.workflowStatus === WorkflowStatus.PENDING) { + return + } + return }, width: 176, minWidth: 176, From 361469046b62ea3329ba29ff1a53ed23ad806a1a Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Tue, 10 Dec 2024 13:42:22 +0800 Subject: [PATCH 08/35] feat: add csv download col --- .../responses/ResponsesPage/storage/types.ts | 3 ++ .../storage/useDecryptionWorkers.ts | 4 ++ .../storage/utils/CsvRecord.class.ts | 3 +- .../EncryptedResponseCsvGenerator.ts | 41 ++++++++++++++--- .../storage/worker/decryption.worker.ts | 10 +++++ shared/types/submission.ts | 3 ++ src/app/models/submission.server.model.ts | 32 ++------------ .../submission/submission.controller.ts | 4 +- .../modules/submission/submission.service.ts | 33 ++++++++++++++ .../modules/submission/submission.utils.ts | 44 +++++++++++++++++-- src/types/submission.ts | 3 ++ 11 files changed, 141 insertions(+), 39 deletions(-) diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/types.ts b/frontend/src/features/admin-form/responses/ResponsesPage/storage/types.ts index 1d0f284783..16171dc27e 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/types.ts +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/types.ts @@ -2,6 +2,8 @@ import { FormField } from '@opengovsg/formsg-sdk/dist/types' import { Remote } from 'comlink' import { SetRequired } from 'type-fest' +import { SubmissionMrfMetadata } from '~shared/types' + import { CsvRecord } from './utils/CsvRecord.class' import { DecryptionWorkerApi } from './worker/decryption.worker' @@ -23,6 +25,7 @@ export type CsvRecordData = FormField export type DecryptedSubmissionData = { created: string submissionId: string + mrfMeta?: SubmissionMrfMetadata record: CsvRecordData[] } diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/useDecryptionWorkers.ts b/frontend/src/features/admin-form/responses/ResponsesPage/storage/useDecryptionWorkers.ts index 31ad9f1013..fb0cf76f42 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/useDecryptionWorkers.ts +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/useDecryptionWorkers.ts @@ -48,6 +48,8 @@ export type DownloadEncryptedParams = EncryptedResponsesStreamParams & { secretKey: string // Number of responses to expect to download. responsesCount: number + // Used to determine if we should add MRF related columns to the CSV. + isMrf: boolean } interface UseDecryptionWorkersProps { onProgress: (progress: number) => void @@ -104,6 +106,7 @@ const useDecryptionWorkers = ({ secretKey, endDate, startDate, + isMrf, }: DownloadEncryptedParams) => { if (!adminForm || !responsesCount) { return Promise.resolve({ @@ -158,6 +161,7 @@ const useDecryptionWorkers = ({ const csvGenerator = new EncryptedResponseCsvGenerator( responsesCount, NUM_OF_METADATA_ROWS, + isMrf, ) const stream = await getEncryptedResponsesStream( diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/CsvRecord.class.ts b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/CsvRecord.class.ts index 2d550aa81a..6d46c60989 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/CsvRecord.class.ts +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/CsvRecord.class.ts @@ -1,6 +1,6 @@ import { cloneDeep } from 'lodash' -import { SubmissionPaymentDto } from '~shared/types' +import { SubmissionMrfMetadata, SubmissionPaymentDto } from '~shared/types' import { getPaymentDataView } from '~features/admin-form/responses/common/utils/getPaymentDataView' @@ -34,6 +34,7 @@ export class CsvRecord { public form: string, public hostOrigin: string, public paymentData?: SubmissionPaymentDto, + public mrfData?: SubmissionMrfMetadata, ) { this.#statusMessage = status this.#record = [] diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts index 9a042b19fc..735a7b7700 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts @@ -15,14 +15,28 @@ type UnprocessedRecord = Merge< { record: Dictionary } > +const MRF_CSV_HEADERS = [ + 'Response ID', + 'Status', + 'Current step', + 'Timestamp of first response', +] + +const NON_MRF_CSV_HEADERS = ['Response ID', 'Timestamp'] + export class EncryptedResponseCsvGenerator extends CsvGenerator { hasBeenProcessed: boolean hasBeenSorted: boolean fieldIdToQuestion: Map fieldIdToNumCols: Record unprocessed: UnprocessedRecord[] + isMrf: boolean - constructor(expectedNumberOfRecords: number, numOfMetaDataRows: number) { + constructor( + expectedNumberOfRecords: number, + numOfMetaDataRows: number, + isMrf: boolean, + ) { super(expectedNumberOfRecords, numOfMetaDataRows) this.hasBeenProcessed = false @@ -30,6 +44,7 @@ export class EncryptedResponseCsvGenerator extends CsvGenerator { this.fieldIdToQuestion = new Map() this.fieldIdToNumCols = {} this.unprocessed = [] + this.isMrf = isMrf } /** @@ -43,7 +58,11 @@ export class EncryptedResponseCsvGenerator extends CsvGenerator { * Extracts information from input record, rearranges record and then adds an UnprocessedRecord to `this.unprocessed` * @throws Error when trying to convert record into a response instance. Should be caught in submissions client factory. */ - addRecord({ record, created, submissionId }: DecryptedSubmissionData): void { + addRecord({ + record, + created, + ...otherSubmissionProperties + }: DecryptedSubmissionData): void { // First pass, create object with { [fieldId]: question } from // decryptedContent to get all the questions. const fieldRecords = record.map((content) => { @@ -75,8 +94,8 @@ export class EncryptedResponseCsvGenerator extends CsvGenerator { // Rearrange record to be an object identified by field ID. this.unprocessed.push({ created, - submissionId, record: keyBy(fieldRecords, (fieldRecord) => fieldRecord.id), + ...otherSubmissionProperties, }) } @@ -89,7 +108,7 @@ export class EncryptedResponseCsvGenerator extends CsvGenerator { if (this.hasBeenProcessed) return // Create a header row in CSV using the fieldIdToQuestion map. - const headers = ['Response ID', 'Timestamp'] + const headers = this.isMrf ? MRF_CSV_HEADERS : NON_MRF_CSV_HEADERS this.fieldIdToQuestion.forEach((value, fieldId) => { for (let i = 0; i < this.fieldIdToNumCols[fieldId]; i++) { headers.push(value.question) @@ -100,6 +119,18 @@ export class EncryptedResponseCsvGenerator extends CsvGenerator { // Craft a new csv row for each unprocessed record // O(qn), where q = number of unique questions, n = number of submissions. this.unprocessed.forEach((up) => { + const row = [up.submissionId] + + if (this.isMrf) { + row.push(up.mrfMeta?.workflowStatus ?? '') + const currentStepString = + up.mrfMeta?.workflowCurrentStepNumber && + up.mrfMeta.workflowNumTotalSteps + ? `Step ${up.mrfMeta?.workflowCurrentStepNumber} of ${up.mrfMeta?.workflowNumTotalSteps}` + : '' + row.push(currentStepString) + } + const formattedDate = isValid(parseISO(up.created)) ? formatInTimeZone( up.created, @@ -107,7 +138,7 @@ export class EncryptedResponseCsvGenerator extends CsvGenerator { 'dd MMM yyyy hh:mm:ss a', ) : up.created - const row = [up.submissionId, formattedDate] + row.push(formattedDate) this.fieldIdToQuestion.forEach((_question, fieldId) => { const numCols = this.fieldIdToNumCols[fieldId] diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/worker/decryption.worker.ts b/frontend/src/features/admin-form/responses/ResponsesPage/storage/worker/decryption.worker.ts index 86ac630891..fd976406df 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/worker/decryption.worker.ts +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/worker/decryption.worker.ts @@ -2,6 +2,8 @@ import { expose } from 'comlink' import { formatInTimeZone } from 'date-fns-tz' import PQueue from 'p-queue' +import { MultirespondentSubmissionStreamDto } from '~shared/types/submission' + import formsgSdk from '~utils/formSdk' import { @@ -99,6 +101,14 @@ async function decryptIntoCsv( submission.submissionType === SubmissionType.Encrypt ? submission.payment : undefined, + submission.submissionType === SubmissionType.Multirespondent + ? { + workflowStatus: (submission as MultirespondentSubmissionStreamDto) + .workflowStatus, + workflowCurrentStepNumber: submission.workflowStep + 1, + workflowNumTotalSteps: submission.numTotalSteps, + } + : undefined, ) try { let decryptedSubmission, submissionSecretKey diff --git a/shared/types/submission.ts b/shared/types/submission.ts index 361101764f..15b6002f69 100644 --- a/shared/types/submission.ts +++ b/shared/types/submission.ts @@ -223,10 +223,13 @@ export const MultirespondentSubmissionStreamDto = encryptedContent: true, version: true, mrfVersion: true, + workflowStep: true, }).extend({ attachmentMetadata: z.record(z.string()), _id: SubmissionId, created: DateString, + workflowStatus: z.nativeEnum(WorkflowStatus), + numTotalSteps: z.number(), }) export type MultirespondentSubmissionStreamDto = z.infer< diff --git a/src/app/models/submission.server.model.ts b/src/app/models/submission.server.model.ts index df50f1a64f..40eb281cad 100644 --- a/src/app/models/submission.server.model.ts +++ b/src/app/models/submission.server.model.ts @@ -37,6 +37,7 @@ import { WebhookView, } from '../../types' import { getPaymentWebhookEventObject } from '../modules/payments/payment.service.utils' +import { getMrfSubmissionWorkflowStatus } from '../modules/submission/submission.utils' import { createQueryWithDateParam } from '../utils/date' import { FORM_SCHEMA_ID } from './form.server.model' @@ -687,6 +688,8 @@ MultirespondentSubmissionSchema.statics.getSubmissionCursorByFormId = function ( form_fields: 1, form_logics: 1, workflow: 1, + workflowStep: 1, + submittedSteps: 1, encryptedSubmissionSecretKey: 1, encryptedContent: 1, attachmentMetadata: 1, @@ -785,33 +788,6 @@ export const getMultirespondentSubmissionModel = ( >(SubmissionType.Multirespondent) } -const getWorkflowStatus = ( - submittedSteps: SubmittedStep[], - numTotalSteps: number, -): WorkflowStatus | undefined => { - if (submittedSteps.length <= 0 || numTotalSteps <= 0) { - // NOTE: this occurs when no steps are recorded for submissions prior to this change or when no workflow is defined. - return undefined - } - const latestSubmittedStep = submittedSteps[submittedSteps.length - 1] - if ( - latestSubmittedStep.isApproval && - latestSubmittedStep.status === WorkflowStatus.REJECTED - ) { - return WorkflowStatus.REJECTED - } - if (submittedSteps.length === numTotalSteps) { - if ( - latestSubmittedStep.isApproval && - latestSubmittedStep.status === WorkflowStatus.APPROVED - ) { - return WorkflowStatus.APPROVED - } - return WorkflowStatus.COMPLETED - } - return WorkflowStatus.PENDING -} - const buildSubmissionMetadata = ( result: MetadataAggregateResult, currentNumber: number, @@ -841,7 +817,7 @@ const buildSubmissionMetadata = ( ? { workflowCurrentStepNumber: mrfMeta.workflowStep + 1 ?? 0, // need to add 1 as workflowStep is 0-indexed workflowNumTotalSteps: mrfMeta.workflow?.length ?? 0, - workflowStatus: getWorkflowStatus( + workflowStatus: getMrfSubmissionWorkflowStatus( mrfMeta.submittedSteps ?? [], mrfMeta.workflow?.length ?? 0, ), diff --git a/src/app/modules/submission/submission.controller.ts b/src/app/modules/submission/submission.controller.ts index 04e9ef2af3..3858134a5c 100644 --- a/src/app/modules/submission/submission.controller.ts +++ b/src/app/modules/submission/submission.controller.ts @@ -30,6 +30,7 @@ import { createMultirespondentSubmissionDto } from './multirespondent-submission import { InvalidSubmissionTypeError } from './submission.errors' import { addPaymentDataStream, + buildMrfMetadata, getEncryptedSubmissionData, getQuarantinePresignedPostData, getSubmissionCursor, @@ -328,7 +329,7 @@ export const streamEncryptedResponses: ControllerHandler< level: PermissionLevel.Read, }), ) - // Step 3: Check whether form is encrypt mode. + // Step 3: Check whether form is encrypt or multirespondent mode. .andThen(checkFormIsEncryptModeOrMultirespondent) // Step 4: Retrieve submissions cursor. .andThen((form) => @@ -370,6 +371,7 @@ export const streamEncryptedResponses: ControllerHandler< urlValidDuration: (req.session?.cookie.maxAge ?? 0) / 1000, }), ) + .pipe(buildMrfMetadata()) // TODO: Can we include this within the cursor query as aggregation pipeline // instead, so that we make one query to mongo rather than two. .pipe(addPaymentDataStream()) diff --git a/src/app/modules/submission/submission.service.ts b/src/app/modules/submission/submission.service.ts index 3f4a7e722f..3ef39462f9 100644 --- a/src/app/modules/submission/submission.service.ts +++ b/src/app/modules/submission/submission.service.ts @@ -17,7 +17,9 @@ import { FormResponseMode, SubmissionMetadata, SubmissionMetadataList, + SubmissionMrfMetadata, SubmissionPaymentDto, + SubmissionType, } from '../../../../shared/types' import { EmailRespondentConfirmationField, @@ -26,6 +28,7 @@ import { IMultirespondentSubmissionModel, IPopulatedForm, ISubmissionSchema, + MultirespondentSubmissionCursorData, StorageModeSubmissionCursorData, SubmissionData, } from '../../../types' @@ -89,6 +92,7 @@ import { fileSizeLimitBytes, getEncryptedSubmissionModelByResponseMode, getInvalidFileExtensions, + getMrfSubmissionWorkflowStatus, mapAttachmentsFromResponses, } from './submission.utils' @@ -800,6 +804,35 @@ export const getSubmissionCursor = ( ) } +export const buildMrfMetadata = (): Transform => { + return new Transform({ + objectMode: true, + transform: async ( + data: + | StorageModeSubmissionCursorData + | MultirespondentSubmissionCursorData, + _encoding, + callback, + ) => { + if (data.submissionType === SubmissionType.Multirespondent) { + const { workflow, workflowStep, submittedSteps, ...rest } = data + return callback(null, { + ...rest, + mrfMeta: { + workflowCurrentStepNumber: workflowStep, + workflowNumTotalSteps: workflow.length, + workflowStatus: getMrfSubmissionWorkflowStatus( + submittedSteps ?? [], + workflow.length, + ), + } as SubmissionMrfMetadata, + }) + } + return callback(null, data) + }, + }) +} + /** * Returns a Transform pipeline that transforms all attachment metadata of each * data chunk from the object path to the S3 signed URL so it can be retrieved diff --git a/src/app/modules/submission/submission.utils.ts b/src/app/modules/submission/submission.utils.ts index 495fc5f91d..711ad9de73 100644 --- a/src/app/modules/submission/submission.utils.ts +++ b/src/app/modules/submission/submission.utils.ts @@ -25,6 +25,8 @@ import { MyInfoAttribute, SubmissionAttachment, SubmissionAttachmentsMap, + SubmittedStep, + WorkflowStatus, } from '../../../../shared/types' import * as FileValidation from '../../../../shared/utils/file-validation' import { @@ -147,10 +149,6 @@ import { const logger = createLoggerWithLabel(module) -const EncryptSubmissionModel = getEncryptSubmissionModel(mongoose) -const MultirespondentSubmissionModel = - getMultirespondentSubmissionModel(mongoose) - type ResponseModeFilterParam = { fieldType: BasicField } @@ -402,6 +400,10 @@ export const getEncryptedSubmissionModelByResponseMode = ( IEncryptSubmissionModel | IMultirespondentSubmissionModel, ResponseModeError > => { + const EncryptSubmissionModel = getEncryptSubmissionModel(mongoose) + const MultirespondentSubmissionModel = + getMultirespondentSubmissionModel(mongoose) + switch (responseMode) { case FormResponseMode.Encrypt: return ok(EncryptSubmissionModel) @@ -817,3 +819,37 @@ export const getCookieNameByAuthType = ( return JwtName[authType] } } + +/** + * Determines the workflow status of a submission based on the submitted steps and the total number of steps. + * @param submittedSteps - The submitted steps of the submission. + * @param numTotalSteps - The total number of steps in the workflow. + * @returns The workflow status of the submission based on the enum `WorkflowStatus`. + * Otherwise, returns `undefined` if no submitted steps or total steps are found or when no workflow has been defined by the form admin. + */ +export const getMrfSubmissionWorkflowStatus = ( + submittedSteps: SubmittedStep[], + numTotalSteps: number, +): WorkflowStatus | undefined => { + if (submittedSteps.length <= 0 || numTotalSteps <= 0) { + // NOTE: this occurs when no steps are recorded for submissions prior to this change or when no workflow is defined. + return undefined + } + const latestSubmittedStep = submittedSteps[submittedSteps.length - 1] + if ( + latestSubmittedStep.isApproval && + latestSubmittedStep.status === WorkflowStatus.REJECTED + ) { + return WorkflowStatus.REJECTED + } + if (submittedSteps.length === numTotalSteps) { + if ( + latestSubmittedStep.isApproval && + latestSubmittedStep.status === WorkflowStatus.APPROVED + ) { + return WorkflowStatus.APPROVED + } + return WorkflowStatus.COMPLETED + } + return WorkflowStatus.PENDING +} diff --git a/src/types/submission.ts b/src/types/submission.ts index be81be6f82..66731aba07 100644 --- a/src/types/submission.ts +++ b/src/types/submission.ts @@ -149,6 +149,9 @@ export type MultirespondentSubmissionCursorData = Pick< | 'id' | 'version' | 'mrfVersion' + | 'workflowStep' + | 'submittedSteps' + | 'workflow' > & { attachmentMetadata?: Record } & Document export type SubmissionCursorData = From 936c973a89a3a14b534ff7b39b664a89807e0279 Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Tue, 10 Dec 2024 14:02:52 +0800 Subject: [PATCH 09/35] fix: add define global --- frontend/vite.config.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index c4277dc150..d6476bcfd4 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -61,5 +61,10 @@ export default defineConfig(() => { plugins: () => [tsconfigPaths()], format: 'es' as const, }, + define: { + // On local dev, global is undefined, causing decryption workers to fail. + // This is a workaround based on https://github.com/vitejs/vite/discussions/5912. + global: 'globalThis', + }, } }) From 009f840890aa9bab0c51c11c982fb52fe076c320 Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Wed, 11 Dec 2024 12:03:42 +0800 Subject: [PATCH 10/35] feat: support cols in csv --- .../ResponsesPage/storage/StorageResponsesProvider.tsx | 3 ++- .../ResponsesPage/storage/useDecryptionWorkers.ts | 2 ++ .../ResponsesPage/storage/utils/CsvRecord.class.ts | 1 + .../ResponsesPage/storage/worker/decryption.worker.ts | 8 ++++---- shared/types/submission.ts | 8 +++++--- src/app/modules/submission/submission.service.ts | 8 ++++++-- 6 files changed, 20 insertions(+), 10 deletions(-) diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/StorageResponsesProvider.tsx b/frontend/src/features/admin-form/responses/ResponsesPage/storage/StorageResponsesProvider.tsx index bcfb934ba7..5d25050e41 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/StorageResponsesProvider.tsx +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/StorageResponsesProvider.tsx @@ -45,8 +45,9 @@ export const StorageResponsesProvider = ({ responsesCount: dateRangeResponsesCount, startDate: dateRange[0], endDate: dateRange[1], + isMrf: form?.responseMode === FormResponseMode.Multirespondent, } - }, [dateRange, dateRangeResponsesCount, secretKey]) + }, [dateRange, dateRangeResponsesCount, secretKey, form?.responseMode]) return ( { if (!adminForm || !responsesCount) { return Promise.resolve({ @@ -439,6 +440,7 @@ const useDecryptionWorkers = ({ const csvGenerator = new EncryptedResponseCsvGenerator( responsesCount, NUM_OF_METADATA_ROWS, + isMrf, ) const stream = await getEncryptedResponsesStream( diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/CsvRecord.class.ts b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/CsvRecord.class.ts index 6d46c60989..44c48d84f1 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/CsvRecord.class.ts +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/CsvRecord.class.ts @@ -136,6 +136,7 @@ export class CsvRecord { this.submissionData = { created: this.created, submissionId: this.id, + mrfMeta: this.mrfData, record: output, } } diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/worker/decryption.worker.ts b/frontend/src/features/admin-form/responses/ResponsesPage/storage/worker/decryption.worker.ts index fd976406df..9e2c2a3b6d 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/worker/decryption.worker.ts +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/worker/decryption.worker.ts @@ -103,10 +103,10 @@ async function decryptIntoCsv( : undefined, submission.submissionType === SubmissionType.Multirespondent ? { - workflowStatus: (submission as MultirespondentSubmissionStreamDto) - .workflowStatus, - workflowCurrentStepNumber: submission.workflowStep + 1, - workflowNumTotalSteps: submission.numTotalSteps, + workflowStatus: submission.mrfMeta.workflowStatus, + workflowCurrentStepNumber: + submission.mrfMeta.workflowCurrentStepNumber, + workflowNumTotalSteps: submission.mrfMeta.workflowNumTotalSteps, } : undefined, ) diff --git a/shared/types/submission.ts b/shared/types/submission.ts index 15b6002f69..4b66f1ee06 100644 --- a/shared/types/submission.ts +++ b/shared/types/submission.ts @@ -223,13 +223,15 @@ export const MultirespondentSubmissionStreamDto = encryptedContent: true, version: true, mrfVersion: true, - workflowStep: true, }).extend({ attachmentMetadata: z.record(z.string()), _id: SubmissionId, created: DateString, - workflowStatus: z.nativeEnum(WorkflowStatus), - numTotalSteps: z.number(), + mrfMeta: z.object({ + workflowCurrentStepNumber: z.number(), + workflowNumTotalSteps: z.number(), + workflowStatus: z.nativeEnum(WorkflowStatus).optional(), + }), }) export type MultirespondentSubmissionStreamDto = z.infer< diff --git a/src/app/modules/submission/submission.service.ts b/src/app/modules/submission/submission.service.ts index 3ef39462f9..00f4ce77eb 100644 --- a/src/app/modules/submission/submission.service.ts +++ b/src/app/modules/submission/submission.service.ts @@ -804,6 +804,9 @@ export const getSubmissionCursor = ( ) } +/** + * Adds mrf metadata to each submission. + */ export const buildMrfMetadata = (): Transform => { return new Transform({ objectMode: true, @@ -816,7 +819,7 @@ export const buildMrfMetadata = (): Transform => { ) => { if (data.submissionType === SubmissionType.Multirespondent) { const { workflow, workflowStep, submittedSteps, ...rest } = data - return callback(null, { + const dataWithMrfMeta = { ...rest, mrfMeta: { workflowCurrentStepNumber: workflowStep, @@ -826,7 +829,8 @@ export const buildMrfMetadata = (): Transform => { workflow.length, ), } as SubmissionMrfMetadata, - }) + } + return callback(null, dataWithMrfMeta) } return callback(null, data) }, From c276fa4f6d51f596bef549af5ecd0761c69bc8a8 Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Wed, 11 Dec 2024 15:21:02 +0800 Subject: [PATCH 11/35] fix: workflowStep being 0 is defined as falsy --- src/app/models/submission.server.model.ts | 15 ++++------ .../submission/submission.controller.ts | 4 +-- .../modules/submission/submission.service.ts | 18 +++++------- .../modules/submission/submission.utils.ts | 28 ++++++++++++++++++- 4 files changed, 42 insertions(+), 23 deletions(-) diff --git a/src/app/models/submission.server.model.ts b/src/app/models/submission.server.model.ts index 40eb281cad..dbea0a4d82 100644 --- a/src/app/models/submission.server.model.ts +++ b/src/app/models/submission.server.model.ts @@ -37,7 +37,7 @@ import { WebhookView, } from '../../types' import { getPaymentWebhookEventObject } from '../modules/payments/payment.service.utils' -import { getMrfSubmissionWorkflowStatus } from '../modules/submission/submission.utils' +import { buildMrfMetadata } from '../modules/submission/submission.utils' import { createQueryWithDateParam } from '../utils/date' import { FORM_SCHEMA_ID } from './form.server.model' @@ -814,14 +814,11 @@ const buildSubmissionMetadata = ( } : null, mrf: mrfMeta - ? { - workflowCurrentStepNumber: mrfMeta.workflowStep + 1 ?? 0, // need to add 1 as workflowStep is 0-indexed - workflowNumTotalSteps: mrfMeta.workflow?.length ?? 0, - workflowStatus: getMrfSubmissionWorkflowStatus( - mrfMeta.submittedSteps ?? [], - mrfMeta.workflow?.length ?? 0, - ), - } + ? buildMrfMetadata({ + workflow: mrfMeta.workflow, + workflowStep: mrfMeta.workflowStep, + submittedSteps: mrfMeta.submittedSteps, + }) : null, } } diff --git a/src/app/modules/submission/submission.controller.ts b/src/app/modules/submission/submission.controller.ts index 3858134a5c..8f21268edf 100644 --- a/src/app/modules/submission/submission.controller.ts +++ b/src/app/modules/submission/submission.controller.ts @@ -29,8 +29,8 @@ import { createStorageModeSubmissionDto } from './encrypt-submission/encrypt-sub import { createMultirespondentSubmissionDto } from './multirespondent-submission/multirespondent-submission.utils' import { InvalidSubmissionTypeError } from './submission.errors' import { + addMrfMetadata, addPaymentDataStream, - buildMrfMetadata, getEncryptedSubmissionData, getQuarantinePresignedPostData, getSubmissionCursor, @@ -371,7 +371,7 @@ export const streamEncryptedResponses: ControllerHandler< urlValidDuration: (req.session?.cookie.maxAge ?? 0) / 1000, }), ) - .pipe(buildMrfMetadata()) + .pipe(addMrfMetadata()) // TODO: Can we include this within the cursor query as aggregation pipeline // instead, so that we make one query to mongo rather than two. .pipe(addPaymentDataStream()) diff --git a/src/app/modules/submission/submission.service.ts b/src/app/modules/submission/submission.service.ts index 00f4ce77eb..e29e34ad96 100644 --- a/src/app/modules/submission/submission.service.ts +++ b/src/app/modules/submission/submission.service.ts @@ -17,7 +17,6 @@ import { FormResponseMode, SubmissionMetadata, SubmissionMetadataList, - SubmissionMrfMetadata, SubmissionPaymentDto, SubmissionType, } from '../../../../shared/types' @@ -89,10 +88,10 @@ import { } from './submission.types' import { areAttachmentsMoreThanLimit, + buildMrfMetadata, fileSizeLimitBytes, getEncryptedSubmissionModelByResponseMode, getInvalidFileExtensions, - getMrfSubmissionWorkflowStatus, mapAttachmentsFromResponses, } from './submission.utils' @@ -807,7 +806,7 @@ export const getSubmissionCursor = ( /** * Adds mrf metadata to each submission. */ -export const buildMrfMetadata = (): Transform => { +export const addMrfMetadata = (): Transform => { return new Transform({ objectMode: true, transform: async ( @@ -821,14 +820,11 @@ export const buildMrfMetadata = (): Transform => { const { workflow, workflowStep, submittedSteps, ...rest } = data const dataWithMrfMeta = { ...rest, - mrfMeta: { - workflowCurrentStepNumber: workflowStep, - workflowNumTotalSteps: workflow.length, - workflowStatus: getMrfSubmissionWorkflowStatus( - submittedSteps ?? [], - workflow.length, - ), - } as SubmissionMrfMetadata, + mrfMeta: buildMrfMetadata({ + workflow, + workflowStep, + submittedSteps, + }), } return callback(null, dataWithMrfMeta) } diff --git a/src/app/modules/submission/submission.utils.ts b/src/app/modules/submission/submission.utils.ts index 711ad9de73..e5fa10b6ba 100644 --- a/src/app/modules/submission/submission.utils.ts +++ b/src/app/modules/submission/submission.utils.ts @@ -25,6 +25,7 @@ import { MyInfoAttribute, SubmissionAttachment, SubmissionAttachmentsMap, + SubmissionMrfMetadata, SubmittedStep, WorkflowStatus, } from '../../../../shared/types' @@ -36,6 +37,7 @@ import { IEncryptSubmissionModel, IFormDocument, IMultirespondentSubmissionModel, + IMultirespondentSubmissionSchema, IPopulatedEncryptedForm, IPopulatedForm, IPopulatedMultirespondentForm, @@ -827,7 +829,7 @@ export const getCookieNameByAuthType = ( * @returns The workflow status of the submission based on the enum `WorkflowStatus`. * Otherwise, returns `undefined` if no submitted steps or total steps are found or when no workflow has been defined by the form admin. */ -export const getMrfSubmissionWorkflowStatus = ( +const getMrfSubmissionWorkflowStatus = ( submittedSteps: SubmittedStep[], numTotalSteps: number, ): WorkflowStatus | undefined => { @@ -853,3 +855,27 @@ export const getMrfSubmissionWorkflowStatus = ( } return WorkflowStatus.PENDING } + +/** + * Builds the metadata for a multirespondent form submission. + */ +export const buildMrfMetadata = ({ + workflow, + workflowStep, + submittedSteps, +}: Pick< + IMultirespondentSubmissionSchema, + 'workflow' | 'workflowStep' | 'submittedSteps' +>): SubmissionMrfMetadata => { + const workflowCurrentStepNumber = workflowStep + 1 // since workflowStep is zero indexed. + const workflowNumTotalSteps = workflow.length + const workflowStatus = getMrfSubmissionWorkflowStatus( + submittedSteps ?? [], + workflow.length, + ) + return { + workflowCurrentStepNumber, + workflowNumTotalSteps, + workflowStatus, + } +} From d5f044e4e2de7539b2e3d0bc3c899221a3c1b1c4 Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Wed, 11 Dec 2024 16:06:11 +0800 Subject: [PATCH 12/35] feat: add approved rejected status in individualresponse --- .../responses/AdminSubmissionsService.ts | 4 ++++ .../IndividualResponsePage.tsx | 16 ++++++++++++++++ .../ResponsesTable/ResponsesTable.tsx | 12 ++++++------ .../EncryptedResponseCsvGenerator.ts | 11 ++++++----- .../responses/common/utils/mrfSubmissionView.ts | 8 ++++++++ shared/types/submission.ts | 2 ++ src/app/models/submission.server.model.ts | 1 + .../multirespondent-submission.utils.ts | 6 ++++++ src/types/submission.ts | 1 + 9 files changed, 50 insertions(+), 11 deletions(-) create mode 100644 frontend/src/features/admin-form/responses/common/utils/mrfSubmissionView.ts diff --git a/frontend/src/features/admin-form/responses/AdminSubmissionsService.ts b/frontend/src/features/admin-form/responses/AdminSubmissionsService.ts index 6bfbb5f78b..443266be3e 100644 --- a/frontend/src/features/admin-form/responses/AdminSubmissionsService.ts +++ b/frontend/src/features/admin-form/responses/AdminSubmissionsService.ts @@ -138,6 +138,10 @@ export const getDecryptedSubmissionById = async ({ encryptedSubmission.submissionType === SubmissionType.Encrypt ? encryptedSubmission.payment : undefined, + mrf: + encryptedSubmission.submissionType === SubmissionType.Multirespondent + ? encryptedSubmission.mrfMeta + : undefined, responses, mrfVersion, } diff --git a/frontend/src/features/admin-form/responses/IndividualResponsePage/IndividualResponsePage.tsx b/frontend/src/features/admin-form/responses/IndividualResponsePage/IndividualResponsePage.tsx index 54719d1b86..aa6e7a63bc 100644 --- a/frontend/src/features/admin-form/responses/IndividualResponsePage/IndividualResponsePage.tsx +++ b/frontend/src/features/admin-form/responses/IndividualResponsePage/IndividualResponsePage.tsx @@ -22,6 +22,7 @@ import { useAdminForm } from '~features/admin-form/common/queries' import { FormActivationSvg } from '~features/admin-form/settings/components/FormActivationSvg' import { useUser } from '~features/user/queries' +import { getCurrentStepString } from '../common/utils/mrfSubmissionView' import { SecretKeyVerification } from '../components/SecretKeyVerification' import { useStorageResponsesContext } from '../ResponsesPage/storage' @@ -156,6 +157,21 @@ export const IndividualResponsePage = (): JSX.Element => { isLoading={isLoading} isError={isError} /> + + [] = [ }, { Header: 'Current Step', - accessor: ({ mrf }) => { - if (!(mrf?.workflowCurrentStepNumber && mrf?.workflowNumTotalSteps)) { - return '' - } - return `Step ${mrf.workflowCurrentStepNumber} of ${mrf.workflowNumTotalSteps}` - }, + accessor: ({ mrf }) => + getCurrentStepString( + mrf?.workflowCurrentStepNumber, + mrf?.workflowNumTotalSteps, + ), width: 176, minWidth: 176, maxWidth: 176, diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts index 735a7b7700..3562360a7f 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts @@ -4,6 +4,8 @@ import type { Dictionary } from 'lodash' import { keyBy } from 'lodash' import type { Merge } from 'type-fest' +import { getCurrentStepString } from '~features/admin-form/responses/common/utils/mrfSubmissionView' + import { CsvGenerator } from '../../../../common/utils' import type { DecryptedSubmissionData } from '../../types' import type { Response } from '../csv-response-classes' @@ -123,11 +125,10 @@ export class EncryptedResponseCsvGenerator extends CsvGenerator { if (this.isMrf) { row.push(up.mrfMeta?.workflowStatus ?? '') - const currentStepString = - up.mrfMeta?.workflowCurrentStepNumber && - up.mrfMeta.workflowNumTotalSteps - ? `Step ${up.mrfMeta?.workflowCurrentStepNumber} of ${up.mrfMeta?.workflowNumTotalSteps}` - : '' + const currentStepString = getCurrentStepString( + up.mrfMeta?.workflowCurrentStepNumber, + up.mrfMeta?.workflowNumTotalSteps, + ) row.push(currentStepString) } diff --git a/frontend/src/features/admin-form/responses/common/utils/mrfSubmissionView.ts b/frontend/src/features/admin-form/responses/common/utils/mrfSubmissionView.ts new file mode 100644 index 0000000000..f73084eb62 --- /dev/null +++ b/frontend/src/features/admin-form/responses/common/utils/mrfSubmissionView.ts @@ -0,0 +1,8 @@ +/** Gets the business friendly string for current step of workflow. */ +export const getCurrentStepString = ( + workflowCurrentStepNumber: number | undefined, + workflowNumTotalSteps: number | undefined, +) => + workflowCurrentStepNumber && workflowNumTotalSteps + ? `Step ${workflowCurrentStepNumber} of ${workflowNumTotalSteps}` + : '' diff --git a/shared/types/submission.ts b/shared/types/submission.ts index 4b66f1ee06..20363f3bc6 100644 --- a/shared/types/submission.ts +++ b/shared/types/submission.ts @@ -192,6 +192,8 @@ export type MultirespondentSubmissionDto = SubmissionDtoBase & { version: number mrfVersion: number + + mrfMeta: SubmissionMrfMetadata } export type SubmissionDto = diff --git a/src/app/models/submission.server.model.ts b/src/app/models/submission.server.model.ts index dbea0a4d82..28457c9711 100644 --- a/src/app/models/submission.server.model.ts +++ b/src/app/models/submission.server.model.ts @@ -731,6 +731,7 @@ MultirespondentSubmissionSchema.statics.findEncryptedSubmissionById = function ( version: 1, workflowStep: 1, mrfVersion: 1, + submittedSteps: 1, }) .exec() } diff --git a/src/app/modules/submission/multirespondent-submission/multirespondent-submission.utils.ts b/src/app/modules/submission/multirespondent-submission/multirespondent-submission.utils.ts index b405842203..b72b172594 100644 --- a/src/app/modules/submission/multirespondent-submission/multirespondent-submission.utils.ts +++ b/src/app/modules/submission/multirespondent-submission/multirespondent-submission.utils.ts @@ -25,6 +25,7 @@ import { ProcessingError, ValidateFieldErrorV3, } from '../submission.errors' +import { buildMrfMetadata } from '../submission.utils' /** * Creates and returns a MultirespondentSubmissionDto object from submissionData and @@ -52,6 +53,11 @@ export const createMultirespondentSubmissionDto = ( version: submissionData.version, workflowStep: submissionData.workflowStep, mrfVersion: submissionData.mrfVersion, + mrfMeta: buildMrfMetadata({ + workflow: submissionData.workflow, + workflowStep: submissionData.workflowStep, + submittedSteps: submissionData.submittedSteps, + }), } } diff --git a/src/types/submission.ts b/src/types/submission.ts index 66731aba07..cc6e13295b 100644 --- a/src/types/submission.ts +++ b/src/types/submission.ts @@ -186,6 +186,7 @@ export type MultirespondentSubmissionData = { | 'version' | 'workflowStep' | 'mrfVersion' + | 'submittedSteps' > & Document From 6a28c16b963a07f0564cb3fdd7a52868b62ca861 Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Wed, 11 Dec 2024 16:13:46 +0800 Subject: [PATCH 13/35] feat: map workflow status to business friendly status --- .../IndividualResponsePage.tsx | 7 +++++-- .../EncryptedResponseCsvGenerator.ts | 10 ++++++++-- .../common/utils/mrfSubmissionView.ts | 18 ++++++++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/frontend/src/features/admin-form/responses/IndividualResponsePage/IndividualResponsePage.tsx b/frontend/src/features/admin-form/responses/IndividualResponsePage/IndividualResponsePage.tsx index aa6e7a63bc..753321b64f 100644 --- a/frontend/src/features/admin-form/responses/IndividualResponsePage/IndividualResponsePage.tsx +++ b/frontend/src/features/admin-form/responses/IndividualResponsePage/IndividualResponsePage.tsx @@ -22,7 +22,10 @@ import { useAdminForm } from '~features/admin-form/common/queries' import { FormActivationSvg } from '~features/admin-form/settings/components/FormActivationSvg' import { useUser } from '~features/user/queries' -import { getCurrentStepString } from '../common/utils/mrfSubmissionView' +import { + getCurrentStepString, + getStatusFromWorkflowStatus, +} from '../common/utils/mrfSubmissionView' import { SecretKeyVerification } from '../components/SecretKeyVerification' import { useStorageResponsesContext } from '../ResponsesPage/storage' @@ -159,7 +162,7 @@ export const IndividualResponsePage = (): JSX.Element => { /> diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts index 3562360a7f..676ccf600f 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts @@ -4,7 +4,10 @@ import type { Dictionary } from 'lodash' import { keyBy } from 'lodash' import type { Merge } from 'type-fest' -import { getCurrentStepString } from '~features/admin-form/responses/common/utils/mrfSubmissionView' +import { + getCurrentStepString, + getStatusFromWorkflowStatus, +} from '~features/admin-form/responses/common/utils/mrfSubmissionView' import { CsvGenerator } from '../../../../common/utils' import type { DecryptedSubmissionData } from '../../types' @@ -124,7 +127,10 @@ export class EncryptedResponseCsvGenerator extends CsvGenerator { const row = [up.submissionId] if (this.isMrf) { - row.push(up.mrfMeta?.workflowStatus ?? '') + const mrfSubmissionStatus = getStatusFromWorkflowStatus( + up.mrfMeta?.workflowStatus, + ) + row.push(mrfSubmissionStatus) const currentStepString = getCurrentStepString( up.mrfMeta?.workflowCurrentStepNumber, up.mrfMeta?.workflowNumTotalSteps, diff --git a/frontend/src/features/admin-form/responses/common/utils/mrfSubmissionView.ts b/frontend/src/features/admin-form/responses/common/utils/mrfSubmissionView.ts index f73084eb62..4b0c921a80 100644 --- a/frontend/src/features/admin-form/responses/common/utils/mrfSubmissionView.ts +++ b/frontend/src/features/admin-form/responses/common/utils/mrfSubmissionView.ts @@ -1,3 +1,5 @@ +import { WorkflowStatus } from '~shared/types' + /** Gets the business friendly string for current step of workflow. */ export const getCurrentStepString = ( workflowCurrentStepNumber: number | undefined, @@ -6,3 +8,19 @@ export const getCurrentStepString = ( workflowCurrentStepNumber && workflowNumTotalSteps ? `Step ${workflowCurrentStepNumber} of ${workflowNumTotalSteps}` : '' + +/** Gets the business friendly string for MRF submission status. */ +export const getStatusFromWorkflowStatus = ( + workflowStatus: WorkflowStatus | undefined, +) => { + switch (workflowStatus) { + case WorkflowStatus.COMPLETED: + case WorkflowStatus.APPROVED: + case WorkflowStatus.REJECTED: + return 'Completed' + case WorkflowStatus.PENDING: + return 'Pending' + default: + return '' + } +} From 8a8ddac18af3dda9c03f69f4a9cc0146644212a3 Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Wed, 11 Dec 2024 16:42:02 +0800 Subject: [PATCH 14/35] fix: change imports to relative --- .../UnlockedResponses/ResponsesTable/ResponsesTable.tsx | 2 +- .../EncryptedResponseCsvGenerator.test.ts | 1 + .../EncryptedResponseCsvGenerator.ts | 5 ++--- .../ResponsesPage/storage/worker/decryption.worker.ts | 2 -- src/app/models/submission.server.model.ts | 2 -- 5 files changed, 4 insertions(+), 8 deletions(-) diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx b/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx index c866f64876..2b4df13e8c 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx @@ -30,8 +30,8 @@ import { centsToDollars } from '~shared/utils/payments' import Badge from '~components/Badge' import { useAdminForm } from '~features/admin-form/common/queries' -import { getCurrentStepString } from '~features/admin-form/responses/common/utils/mrfSubmissionView' +import { getCurrentStepString } from '../../../../common/utils/mrfSubmissionView' import { useUnlockedResponses } from '../UnlockedResponsesProvider' import { getNetAmount } from './utils' diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.test.ts b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.test.ts index 9529a50b06..e36542e5af 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.test.ts +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.test.ts @@ -101,6 +101,7 @@ describe('EncryptedResponseCsvGenerator', () => { generator = new EncryptedResponseCsvGenerator( mockExpectedNumberOfRecords, mockNumOfMetaDataRows, + false, ) }) diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts index 676ccf600f..deca0ceb92 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts @@ -4,12 +4,11 @@ import type { Dictionary } from 'lodash' import { keyBy } from 'lodash' import type { Merge } from 'type-fest' +import { CsvGenerator } from '../../../../common/utils' import { getCurrentStepString, getStatusFromWorkflowStatus, -} from '~features/admin-form/responses/common/utils/mrfSubmissionView' - -import { CsvGenerator } from '../../../../common/utils' +} from '../../../../common/utils/mrfSubmissionView' import type { DecryptedSubmissionData } from '../../types' import type { Response } from '../csv-response-classes' import { getDecryptedResponseInstance } from '../getDecryptedResponseInstance' diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/worker/decryption.worker.ts b/frontend/src/features/admin-form/responses/ResponsesPage/storage/worker/decryption.worker.ts index 9e2c2a3b6d..6ef4060bb1 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/worker/decryption.worker.ts +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/worker/decryption.worker.ts @@ -2,8 +2,6 @@ import { expose } from 'comlink' import { formatInTimeZone } from 'date-fns-tz' import PQueue from 'p-queue' -import { MultirespondentSubmissionStreamDto } from '~shared/types/submission' - import formsgSdk from '~utils/formSdk' import { diff --git a/src/app/models/submission.server.model.ts b/src/app/models/submission.server.model.ts index 28457c9711..dfc7ba013d 100644 --- a/src/app/models/submission.server.model.ts +++ b/src/app/models/submission.server.model.ts @@ -11,9 +11,7 @@ import { MyInfoAttribute, SubmissionMetadata, SubmissionType, - SubmittedStep, WebhookResponse, - WorkflowStatus, } from '../../../shared/types' import { FindFormsWithSubsAboveResult, From a8bdd88e9371705db2be1ff6c885bb04cdd27e6d Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Fri, 13 Dec 2024 11:42:59 +0800 Subject: [PATCH 15/35] fix: failing unit tests --- shared/types/submission.ts | 12 +- ...respondent-submission.server.model.spec.ts | 113 ++++++++++++++++-- .../__tests__/submission.server.model.spec.ts | 1 + src/app/models/submission.server.model.ts | 2 +- ...tirespondent-submission.controller.spec.ts | 2 +- .../multirespondent-submission.utils.spec.ts | 36 +++++- 6 files changed, 148 insertions(+), 18 deletions(-) diff --git a/shared/types/submission.ts b/shared/types/submission.ts index 20363f3bc6..410b01ba9e 100644 --- a/shared/types/submission.ts +++ b/shared/types/submission.ts @@ -254,11 +254,13 @@ export type SubmissionPaymentMetadata = { email: string } | null -export type SubmissionMrfMetadata = { - workflowCurrentStepNumber: number - workflowNumTotalSteps: number - workflowStatus: WorkflowStatus | undefined // `undefined` is due to submissions before this PR not storing this value -} | null +export type SubmissionMrfMetadata = + | { + workflowCurrentStepNumber: number + workflowNumTotalSteps: number + workflowStatus: WorkflowStatus | undefined // `undefined` is due to submissions before this PR not storing this value + } + | undefined export type SubmissionMetadata = { number: number diff --git a/src/app/models/__tests__/multirespondent-submission.server.model.spec.ts b/src/app/models/__tests__/multirespondent-submission.server.model.spec.ts index 663326d672..63c855c9f1 100644 --- a/src/app/models/__tests__/multirespondent-submission.server.model.spec.ts +++ b/src/app/models/__tests__/multirespondent-submission.server.model.spec.ts @@ -3,7 +3,13 @@ import { ObjectId } from 'bson' import { pick, times } from 'lodash' import moment from 'moment-timezone' import mongoose from 'mongoose' -import { SubmissionMetadata, SubmissionType } from 'shared/types' +import { + BasicField, + SubmissionMetadata, + SubmissionType, + WorkflowStatus, + WorkflowType, +} from 'shared/types' import getSubmissionModel, { getEmailSubmissionModel, @@ -24,6 +30,34 @@ describe('Multirespondent Submission Model', () => { const MOCK_ENCRYPTED_SUBMISSION_SECRET_KEY = 'This is an encrypted secret key' const MOCK_ENCRYPTED_CONTENT = 'abcdefg encryptedContent' + const YES_NO_FIELD = { + _id: 'yes_no_field_id', + title: 'Yes or No', + description: '', + required: true, + disabled: false, + fieldType: BasicField.YesNo, + } + const WORKFLOW_STEP_1 = { + _id: 'step_1_id', + workflow_type: WorkflowType.Static, + emails: ['example@example.com'], + edit: [], + } + const WORKFLOW_STEP_2 = { + _id: 'step_2_id', + workflow_type: WorkflowType.Static, + emails: ['example@example.com'], + edit: [YES_NO_FIELD._id], + } + const WORKFLOW_APPROVAL_STEP = { + _id: 'approval_step_id', + workflow_type: WorkflowType.Static, + emails: ['example@example.com'], + edit: [YES_NO_FIELD._id], + approval_field: YES_NO_FIELD._id, + } + describe('Statics', () => { describe('findSingleMetadata', () => { it('should return submission metadata', async () => { @@ -34,15 +68,26 @@ describe('Multirespondent Submission Model', () => { const validSubmission = await MultirespondentSubmission.create({ form: validFormId, submissionType: SubmissionType.Multirespondent, - form_fields: [], + form_fields: [YES_NO_FIELD, WORKFLOW_APPROVAL_STEP], form_logics: [], - workflow: [], + workflow: [WORKFLOW_STEP_1, WORKFLOW_APPROVAL_STEP], submissionPublicKey: MOCK_SUBMISSION_PUBLIC_KEY, encryptedSubmissionSecretKey: MOCK_ENCRYPTED_SUBMISSION_SECRET_KEY, encryptedContent: MOCK_ENCRYPTED_CONTENT, version: 3, created: createdDate, - workflowStep: 0, + workflowStep: 1, + submittedSteps: [ + { + isApproval: false, + submittedAt: '2024-01-01T00:00:00.000Z', + }, + { + isApproval: true, + status: WorkflowStatus.REJECTED, + submittedAt: '2024-01-01T00:00:00.000Z', + }, + ], }) // Act @@ -59,6 +104,11 @@ describe('Multirespondent Submission Model', () => { submissionTime: moment(createdDate) .tz('Asia/Singapore') .format('Do MMM YYYY, h:mm:ss a'), + mrf: { + workflowCurrentStepNumber: 2, + workflowNumTotalSteps: 2, + workflowStatus: WorkflowStatus.REJECTED, + }, } expect(result).toEqual(expected) }) @@ -118,13 +168,19 @@ describe('Multirespondent Submission Model', () => { submissionType: SubmissionType.Multirespondent, form_fields: [], form_logics: [], - workflow: [], + workflow: [WORKFLOW_STEP_1], submissionPublicKey: MOCK_SUBMISSION_PUBLIC_KEY, encryptedSubmissionSecretKey: MOCK_ENCRYPTED_SUBMISSION_SECRET_KEY, encryptedContent: MOCK_ENCRYPTED_CONTENT, version: 3, created: MOCK_CREATED_DATES_ASC[idx], workflowStep: 0, + submittedSteps: [ + { + isApproval: false, + submittedAt: '2024-01-01T00:00:00.000Z', + }, + ], }), ) const validSubmissions: IMultirespondentSubmissionSchema[] = @@ -146,6 +202,11 @@ describe('Multirespondent Submission Model', () => { submissionTime: moment(data.created) .tz('Asia/Singapore') .format('Do MMM YYYY, h:mm:ss a'), + mrf: { + workflowCurrentStepNumber: 1, + workflowNumTotalSteps: 1, + workflowStatus: WorkflowStatus.COMPLETED, + }, })) .reverse(), } @@ -161,13 +222,24 @@ describe('Multirespondent Submission Model', () => { submissionType: SubmissionType.Multirespondent, form_fields: [], form_logics: [], - workflow: [], + workflow: [WORKFLOW_STEP_1, WORKFLOW_APPROVAL_STEP], submissionPublicKey: MOCK_SUBMISSION_PUBLIC_KEY, encryptedSubmissionSecretKey: MOCK_ENCRYPTED_SUBMISSION_SECRET_KEY, encryptedContent: MOCK_ENCRYPTED_CONTENT, version: 3, created: MOCK_CREATED_DATES_ASC[idx], - workflowStep: 0, + workflowStep: 1, + submittedSteps: [ + { + isApproval: false, + submittedAt: '2024-01-01T00:00:00.000Z', + }, + { + isApproval: true, + status: WorkflowStatus.APPROVED, + submittedAt: '2024-01-01T00:00:00.000Z', + }, + ], }), ) const validSubmissions: IMultirespondentSubmissionSchema[] = @@ -194,6 +266,11 @@ describe('Multirespondent Submission Model', () => { submissionTime: moment(secondSubmission.created) .tz('Asia/Singapore') .format('Do MMM YYYY, h:mm:ss a'), + mrf: { + workflowCurrentStepNumber: 2, + workflowNumTotalSteps: 2, + workflowStatus: WorkflowStatus.APPROVED, + }, }, ], } @@ -209,13 +286,23 @@ describe('Multirespondent Submission Model', () => { submissionType: SubmissionType.Multirespondent, form_fields: [], form_logics: [], - workflow: [], + workflow: [WORKFLOW_STEP_1, WORKFLOW_STEP_2], submissionPublicKey: MOCK_SUBMISSION_PUBLIC_KEY, encryptedSubmissionSecretKey: MOCK_ENCRYPTED_SUBMISSION_SECRET_KEY, encryptedContent: MOCK_ENCRYPTED_CONTENT, version: 3, created: MOCK_CREATED_DATES_ASC[idx], - workflowStep: 0, + workflowStep: 1, + submittedSteps: [ + { + isApproval: false, + submittedAt: '2024-01-01T00:00:00.000Z', + }, + { + isApproval: false, + submittedAt: '2024-01-02T00:00:00.000Z', + }, + ], }), ) const validSubmissions: IMultirespondentSubmissionSchema[] = @@ -242,6 +329,11 @@ describe('Multirespondent Submission Model', () => { submissionTime: moment(latestSubmission.created) .tz('Asia/Singapore') .format('Do MMM YYYY, h:mm:ss a'), + mrf: { + workflowCurrentStepNumber: 2, + workflowNumTotalSteps: 2, + workflowStatus: WorkflowStatus.COMPLETED, + }, }, ], } @@ -332,6 +424,8 @@ describe('Multirespondent Submission Model', () => { 'encryptedContent', 'submissionType', 'version', + 'submittedSteps', + 'workflowStep', ) // Native-ify arrays as mongoose documents contain a mongoose-specific array type. expectedSubmission.form_fields = JSON.parse( @@ -417,6 +511,7 @@ describe('Multirespondent Submission Model', () => { 'submissionType', 'version', 'workflowStep', + 'submittedSteps', ) expect(actual).not.toBeNull() expect(actual?.toJSON()).toEqual(expected) diff --git a/src/app/models/__tests__/submission.server.model.spec.ts b/src/app/models/__tests__/submission.server.model.spec.ts index d0da0a1512..d50aa0b480 100644 --- a/src/app/models/__tests__/submission.server.model.spec.ts +++ b/src/app/models/__tests__/submission.server.model.spec.ts @@ -91,6 +91,7 @@ describe('Submission Model', () => { field: new ObjectId(), }, ], + submittedSteps: [], submissionPublicKey: 'This is a public key', encryptedSubmissionSecretKey: 'This is an encrypted secret key', encryptedContent: MOCK_ENCRYPTED_CONTENT, diff --git a/src/app/models/submission.server.model.ts b/src/app/models/submission.server.model.ts index dfc7ba013d..054b538629 100644 --- a/src/app/models/submission.server.model.ts +++ b/src/app/models/submission.server.model.ts @@ -818,7 +818,7 @@ const buildSubmissionMetadata = ( workflowStep: mrfMeta.workflowStep, submittedSteps: mrfMeta.submittedSteps, }) - : null, + : undefined, } } diff --git a/src/app/modules/submission/multirespondent-submission/__tests__/multirespondent-submission.controller.spec.ts b/src/app/modules/submission/multirespondent-submission/__tests__/multirespondent-submission.controller.spec.ts index 33c40537a6..198a5da365 100644 --- a/src/app/modules/submission/multirespondent-submission/__tests__/multirespondent-submission.controller.spec.ts +++ b/src/app/modules/submission/multirespondent-submission/__tests__/multirespondent-submission.controller.spec.ts @@ -331,6 +331,7 @@ describe('multiresponodent-submision.controller', () => { expect( MockMultiRespondentSubmissionService.updateMultiRespondentFormSubmission, ).toHaveBeenCalledOnce() + expect( omit( MockMultiRespondentSubmissionService @@ -338,7 +339,6 @@ describe('multiresponodent-submision.controller', () => { 'logMeta', ), ).toEqual({ - formId: mockFormId, submissionId: mockSubmissionId, encryptedPayload: mockSubmitMrfReq.formsg.encryptedPayload, form: mockSubmitMrfReq.formsg.formDef, diff --git a/src/app/modules/submission/multirespondent-submission/__tests__/multirespondent-submission.utils.spec.ts b/src/app/modules/submission/multirespondent-submission/__tests__/multirespondent-submission.utils.spec.ts index 5b22c65008..e46ccb49b5 100644 --- a/src/app/modules/submission/multirespondent-submission/__tests__/multirespondent-submission.utils.spec.ts +++ b/src/app/modules/submission/multirespondent-submission/__tests__/multirespondent-submission.utils.spec.ts @@ -16,6 +16,7 @@ import { ShortTextResponseV3, SubmissionType, TableResponseV3, + WorkflowStatus, WorkflowType, } from 'shared/types' @@ -41,18 +42,38 @@ import { } from '../multirespondent-submission.utils' describe('multirespondent-submission.utils', () => { + const WORKFLOW_STEP_1 = { + _id: 'step_1_id', + workflow_type: WorkflowType.Static, + emails: ['example@example.com'], + edit: [], + } + describe('createMultirespondentSubmissionDto', () => { it('should create an encrypted submission DTO sucessfully', () => { // Arrange const createdDate = new Date() const submissionData = { + submissionType: SubmissionType.Multirespondent, _id: new ObjectId(), created: createdDate, submissionPublicKey: 'some public key', encryptedSubmissionSecretKey: 'some encrypted secret key', encryptedContent: 'some encrypted content', - submissionType: SubmissionType.Multirespondent, - } as MultirespondentSubmissionData + workflow: [WORKFLOW_STEP_1], + workflowStep: 0, + form_fields: [], + form_logics: [], + attachmentMetadata: {}, + version: 3, + mrfVersion: 3, + submittedSteps: [ + { + isApproval: false, + submittedAt: '2024-01-01T00:00:00.000Z', + }, + ], + } as unknown as MultirespondentSubmissionData const attachmentPresignedUrls = { someSubmissionId: 'some presigned url', } @@ -75,6 +96,17 @@ describe('multirespondent-submission.utils', () => { submissionData.encryptedSubmissionSecretKey, attachmentMetadata: attachmentPresignedUrls, submissionType: SubmissionType.Multirespondent, + workflow: submissionData.workflow, + form_fields: submissionData.form_fields, + form_logics: submissionData.form_logics, + version: submissionData.version, + workflowStep: submissionData.workflowStep, + mrfVersion: submissionData.mrfVersion, + mrfMeta: { + workflowCurrentStepNumber: 1, + workflowNumTotalSteps: 1, + workflowStatus: WorkflowStatus.COMPLETED, + }, }) }) }) From 609ccecc416d032a46039170d4259136ab6dc41a Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Fri, 13 Dec 2024 20:57:58 +0800 Subject: [PATCH 16/35] feat: add datetime validation for submittedAt --- shared/types/submission.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/types/submission.ts b/shared/types/submission.ts index 410b01ba9e..0b0cff4e11 100644 --- a/shared/types/submission.ts +++ b/shared/types/submission.ts @@ -94,7 +94,7 @@ export const ApprovalStatus = z.enum([ const SubmittedNonApprovalStep = z.object({ isApproval: z.literal(false), - submittedAt: z.string(), + submittedAt: z.string().datetime({ precision: 3 }), }) export type SubmittedNonApprovalStep = z.infer From 60d1654d22297a1194f32f7cf7978d83023d0186 Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Fri, 13 Dec 2024 21:39:28 +0800 Subject: [PATCH 17/35] fix: failing FE unit tests --- .../EncryptedResponseCsvGenerator.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts index deca0ceb92..9f0c0cee51 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts @@ -112,7 +112,7 @@ export class EncryptedResponseCsvGenerator extends CsvGenerator { if (this.hasBeenProcessed) return // Create a header row in CSV using the fieldIdToQuestion map. - const headers = this.isMrf ? MRF_CSV_HEADERS : NON_MRF_CSV_HEADERS + const headers = this.isMrf ? [...MRF_CSV_HEADERS] : [...NON_MRF_CSV_HEADERS] this.fieldIdToQuestion.forEach((value, fieldId) => { for (let i = 0; i < this.fieldIdToNumCols[fieldId]; i++) { headers.push(value.question) From 86c46c2fbd9bd2d768061e9e0ebc6b7e6716b872 Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Fri, 13 Dec 2024 21:55:08 +0800 Subject: [PATCH 18/35] feat: resize table cols for mrf to follow storage mode as much as possible --- .../ResponsesTable/ResponsesTable.tsx | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx b/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx index 2b4df13e8c..e8b043a42c 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx @@ -80,7 +80,6 @@ const NON_MRF_RESPONSE_TABLE_COLUMNS: Column[] = [ { Header: 'Timestamp', accessor: 'submissionTime', - minWidth: 250, width: 250, disableResizing: true, }, @@ -174,9 +173,9 @@ const MRF_RESPONSE_TABLE_COLUMNS: Column[] = [ } return }, - width: 176, - minWidth: 176, - maxWidth: 176, + width: 200, + minWidth: 180, + maxWidth: 220, }, { Header: 'Current Step', @@ -185,16 +184,15 @@ const MRF_RESPONSE_TABLE_COLUMNS: Column[] = [ mrf?.workflowCurrentStepNumber, mrf?.workflowNumTotalSteps, ), - width: 176, - minWidth: 176, - maxWidth: 176, + width: 200, + minWidth: 180, + maxWidth: 220, }, { Header: 'Timestamp of first response', accessor: 'submissionTime', - width: 320, - minWidth: 320, - maxWidth: 320, + width: 250, + disableResizing: true }, ] From 177f2fdf0c02e3e8ead0b73d520c44b2647151ac Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Fri, 13 Dec 2024 22:10:36 +0800 Subject: [PATCH 19/35] feat: update copy for first step --- .../IndividualResponsePage.tsx | 28 ++++++++++++++++++- .../ResponsesTable/ResponsesTable.tsx | 7 +++-- .../EncryptedResponseCsvGenerator.ts | 12 ++++++-- .../admin-form/responses/constants.ts | 3 ++ 4 files changed, 43 insertions(+), 7 deletions(-) create mode 100644 frontend/src/features/admin-form/responses/constants.ts diff --git a/frontend/src/features/admin-form/responses/IndividualResponsePage/IndividualResponsePage.tsx b/frontend/src/features/admin-form/responses/IndividualResponsePage/IndividualResponsePage.tsx index 753321b64f..66091c045f 100644 --- a/frontend/src/features/admin-form/responses/IndividualResponsePage/IndividualResponsePage.tsx +++ b/frontend/src/features/admin-form/responses/IndividualResponsePage/IndividualResponsePage.tsx @@ -27,6 +27,11 @@ import { getStatusFromWorkflowStatus, } from '../common/utils/mrfSubmissionView' import { SecretKeyVerification } from '../components/SecretKeyVerification' +import { + MRF_CURRENT_STEP_LABEL, + MRF_FIRST_STEP_TIMESTAMP_LABEL, + MRF_STATUS_LABEL, +} from '../constants' import { useStorageResponsesContext } from '../ResponsesPage/storage' import { DecryptedRow } from './DecryptedRow' @@ -88,6 +93,8 @@ export const IndividualResponsePage = (): JSX.Element => { const { data: form } = useAdminForm() + const isMrf = form?.responseMode === FormResponseMode.Multirespondent + const { user } = useUser() const { secretKey } = useStorageResponsesContext() const { data, isLoading, isError } = useIndividualSubmission() @@ -160,6 +167,25 @@ export const IndividualResponsePage = (): JSX.Element => { isLoading={isLoading} isError={isError} /> + {isMrf ? ( + <> + + + + ) : null} { isError={isError} /> [] = [ maxWidth: 300, }, { - Header: 'Status', + Header: MRF_STATUS_LABEL, accessor: ({ mrf }) => { if (!mrf?.workflowStatus) { return '' @@ -178,7 +179,7 @@ const MRF_RESPONSE_TABLE_COLUMNS: Column[] = [ maxWidth: 220, }, { - Header: 'Current Step', + Header: MRF_CURRENT_STEP_LABEL, accessor: ({ mrf }) => getCurrentStepString( mrf?.workflowCurrentStepNumber, @@ -189,7 +190,7 @@ const MRF_RESPONSE_TABLE_COLUMNS: Column[] = [ maxWidth: 220, }, { - Header: 'Timestamp of first response', + Header: MRF_FIRST_STEP_TIMESTAMP_LABEL, accessor: 'submissionTime', width: 250, disableResizing: true diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts index 9f0c0cee51..b9823885ca 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts @@ -4,6 +4,12 @@ import type { Dictionary } from 'lodash' import { keyBy } from 'lodash' import type { Merge } from 'type-fest' +import { + MRF_CURRENT_STEP_LABEL, + MRF_FIRST_STEP_TIMESTAMP_LABEL, + MRF_STATUS_LABEL, +} from '~features/admin-form/responses/constants' + import { CsvGenerator } from '../../../../common/utils' import { getCurrentStepString, @@ -21,9 +27,9 @@ type UnprocessedRecord = Merge< const MRF_CSV_HEADERS = [ 'Response ID', - 'Status', - 'Current step', - 'Timestamp of first response', + MRF_STATUS_LABEL, + MRF_CURRENT_STEP_LABEL, + MRF_FIRST_STEP_TIMESTAMP_LABEL, ] const NON_MRF_CSV_HEADERS = ['Response ID', 'Timestamp'] diff --git a/frontend/src/features/admin-form/responses/constants.ts b/frontend/src/features/admin-form/responses/constants.ts new file mode 100644 index 0000000000..62fa96b634 --- /dev/null +++ b/frontend/src/features/admin-form/responses/constants.ts @@ -0,0 +1,3 @@ +export const MRF_FIRST_STEP_TIMESTAMP_LABEL = 'Submission time of first step' +export const MRF_CURRENT_STEP_LABEL = 'Current step' +export const MRF_STATUS_LABEL = 'Status' From 75ac07090ec58ef5293133b89f007adc3c602334 Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Fri, 13 Dec 2024 22:31:00 +0800 Subject: [PATCH 20/35] fix: bug where last col does not expand in scrollable --- .../UnlockedResponses/ResponsesTable/ResponsesTable.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx b/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx index ef06db57c2..8b937e5b5e 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx @@ -82,6 +82,7 @@ const NON_MRF_RESPONSE_TABLE_COLUMNS: Column[] = [ Header: 'Timestamp', accessor: 'submissionTime', width: 250, + minWidth: 250, disableResizing: true, }, ] @@ -193,7 +194,8 @@ const MRF_RESPONSE_TABLE_COLUMNS: Column[] = [ Header: MRF_FIRST_STEP_TIMESTAMP_LABEL, accessor: 'submissionTime', width: 250, - disableResizing: true + minWidth: 250, + disableResizing: true, }, ] From 90eb8e57ee8671d3d6b7b62057a0edf047d6775e Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Fri, 13 Dec 2024 22:51:22 +0800 Subject: [PATCH 21/35] feat: add tc for testing the metadata builder --- .../__tests__/submission.utils.spec.ts | 158 +++++++++++++++++- .../multirespondent-submission.utils.spec.ts | 9 +- 2 files changed, 162 insertions(+), 5 deletions(-) diff --git a/src/app/modules/submission/__tests__/submission.utils.spec.ts b/src/app/modules/submission/__tests__/submission.utils.spec.ts index a594bfa240..0cdcc90c99 100644 --- a/src/app/modules/submission/__tests__/submission.utils.spec.ts +++ b/src/app/modules/submission/__tests__/submission.utils.spec.ts @@ -2,10 +2,17 @@ import { ObjectId } from 'bson' import { readFileSync } from 'fs' import { cloneDeep, merge } from 'lodash' -import { BasicField, FormResponseMode } from '../../../../../shared/types' +import { + BasicField, + FormResponseMode, + FormWorkflowStepDto, + WorkflowStatus, + WorkflowType, +} from '../../../../../shared/types' import { SingleAnswerFieldResponse } from '../../../../types' import { areAttachmentsMoreThanLimit, + buildMrfMetadata, getInvalidFileExtensions, getResponseModeFilter, mapAttachmentsFromResponses, @@ -66,6 +73,155 @@ const getResponse = (_id: string, answer: string): SingleAnswerFieldResponse => }) as unknown as SingleAnswerFieldResponse describe('submission.utils', () => { + describe('buildMrfMetadata', () => { + const YES_NO_FIELD = { + _id: 'yes_no_field_id', + title: 'Yes or No', + description: '', + required: true, + disabled: false, + fieldType: BasicField.YesNo, + } + const WORKFLOW_STEP_1: FormWorkflowStepDto = { + _id: 'step_1_id', + workflow_type: WorkflowType.Static, + emails: ['example@example.com'], + edit: [], + } + + const WORKFLOW_STEP_2: FormWorkflowStepDto = { + _id: 'step_2_id', + workflow_type: WorkflowType.Static, + emails: ['example@example.com'], + edit: [YES_NO_FIELD._id], + } + const WORKFLOW_APPROVAL_STEP: FormWorkflowStepDto = { + _id: 'approval_step_id', + workflow_type: WorkflowType.Static, + emails: ['example@example.com'], + edit: [YES_NO_FIELD._id], + approval_field: YES_NO_FIELD._id, + } + + it('should build mrf metadata successfully for pending submission without approval step', () => { + const metadata = buildMrfMetadata({ + workflow: [WORKFLOW_STEP_1, WORKFLOW_STEP_2], + workflowStep: 0, + submittedSteps: [ + { + isApproval: false, + submittedAt: '2024-01-01T00:00:00.000Z', + }, + ], + }) + + expect(metadata).toEqual({ + workflowCurrentStepNumber: 1, + workflowNumTotalSteps: 2, + workflowStatus: WorkflowStatus.PENDING, + }) + }) + + it('should build mrf metadata successfully for completed submission without approval step', () => { + const metadata = buildMrfMetadata({ + workflow: [WORKFLOW_STEP_1, WORKFLOW_STEP_2], + workflowStep: 1, + submittedSteps: [ + { + isApproval: false, + submittedAt: '2024-01-01T00:00:00.000Z', + }, + { + isApproval: false, + submittedAt: '2024-01-02T00:00:00.000Z', + }, + ], + }) + + expect(metadata).toEqual({ + workflowCurrentStepNumber: 2, + workflowNumTotalSteps: 2, + workflowStatus: WorkflowStatus.COMPLETED, + }) + }) + + it('should build mrf metadata successfully for pending submission with approval step', () => { + const metadata = buildMrfMetadata({ + workflow: [WORKFLOW_STEP_1, WORKFLOW_APPROVAL_STEP, WORKFLOW_STEP_2], + workflowStep: 1, + submittedSteps: [ + { + isApproval: false, + submittedAt: '2024-01-01T00:00:00.000Z', + }, + { + isApproval: true, + status: WorkflowStatus.APPROVED, + submittedAt: '2024-01-02T00:00:00.000Z', + }, + ], + }) + + expect(metadata).toEqual({ + workflowCurrentStepNumber: 2, + workflowNumTotalSteps: 3, + workflowStatus: WorkflowStatus.PENDING, + }) + }) + + it('should build mrf metadata successfully for approval submission with approval step', () => { + const metadata = buildMrfMetadata({ + workflow: [WORKFLOW_STEP_1, WORKFLOW_APPROVAL_STEP, WORKFLOW_STEP_2], + workflowStep: 2, + submittedSteps: [ + { + isApproval: false, + submittedAt: '2024-01-01T00:00:00.000Z', + }, + { + isApproval: true, + status: WorkflowStatus.APPROVED, + submittedAt: '2024-01-02T00:00:00.000Z', + }, + { + isApproval: false, + submittedAt: '2024-01-03T00:00:00.000Z', + }, + ], + }) + + expect(metadata).toEqual({ + workflowCurrentStepNumber: 3, + workflowNumTotalSteps: 3, + workflowStatus: WorkflowStatus.APPROVED, + }) + }) + + it('should build mrf metadata successfully for rejected submission with approval step', () => { + const metadata = buildMrfMetadata({ + workflow: [WORKFLOW_STEP_1, WORKFLOW_APPROVAL_STEP, WORKFLOW_STEP_2], + workflowStep: 1, + submittedSteps: [ + { + isApproval: false, + submittedAt: '2024-01-01T00:00:00.000Z', + }, + { + isApproval: true, + status: WorkflowStatus.REJECTED, + submittedAt: '2024-01-02T00:00:00.000Z', + }, + ], + }) + + expect(metadata).toEqual({ + workflowCurrentStepNumber: 2, + workflowNumTotalSteps: 3, + workflowStatus: WorkflowStatus.REJECTED, + }) + }) + }) + describe('getResponseModeFilter', () => { const ALL_FIELD_TYPES = Object.values(BasicField).map((fieldType) => ({ fieldType, diff --git a/src/app/modules/submission/multirespondent-submission/__tests__/multirespondent-submission.utils.spec.ts b/src/app/modules/submission/multirespondent-submission/__tests__/multirespondent-submission.utils.spec.ts index e46ccb49b5..01fc7c6c76 100644 --- a/src/app/modules/submission/multirespondent-submission/__tests__/multirespondent-submission.utils.spec.ts +++ b/src/app/modules/submission/multirespondent-submission/__tests__/multirespondent-submission.utils.spec.ts @@ -10,6 +10,7 @@ import { ChildBirthRecordsResponseV3, EmailResponseV3, FieldResponsesV3, + FormFieldDto, FormWorkflowStepDto, LongTextResponseV3, NumberResponseV3, @@ -42,7 +43,7 @@ import { } from '../multirespondent-submission.utils' describe('multirespondent-submission.utils', () => { - const WORKFLOW_STEP_1 = { + const WORKFLOW_STEP_1: FormWorkflowStepDto = { _id: 'step_1_id', workflow_type: WorkflowType.Static, emails: ['example@example.com'], @@ -140,7 +141,7 @@ describe('multirespondent-submission.utils', () => { const result = validateMrfFieldResponses({ formId: mockFormId, visibleFieldIds: mockVisibleFieldIds, - formFields: mockFormFields, + formFields: mockFormFields as FormFieldDto[], responses: mockResponses, }) @@ -174,7 +175,7 @@ describe('multirespondent-submission.utils', () => { validateMrfFieldResponses({ formId: mockFormId, visibleFieldIds: mockVisibleFieldIds, - formFields: mockFormFields, + formFields: mockFormFields as FormFieldDto[], responses: mockResponses, }) @@ -217,7 +218,7 @@ describe('multirespondent-submission.utils', () => { validateMrfFieldResponses({ formId: mockFormId, visibleFieldIds: mockVisibleFieldIds, - formFields: mockFormFields, + formFields: mockFormFields as FormFieldDto[], responses: mockResponses, }) From 2384693ed39813eb04b83850e2a6620f1349e989 Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Fri, 13 Dec 2024 23:08:32 +0800 Subject: [PATCH 22/35] feat: add tc for appending mrf meta to mrf submission stream --- .../__tests__/submission.service.spec.ts | 90 ++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/src/app/modules/submission/__tests__/submission.service.spec.ts b/src/app/modules/submission/__tests__/submission.service.spec.ts index 6436e1a4dc..673754027b 100644 --- a/src/app/modules/submission/__tests__/submission.service.spec.ts +++ b/src/app/modules/submission/__tests__/submission.service.spec.ts @@ -41,9 +41,12 @@ import { AutoReplyOptions, BasicField, FormResponseMode, + FormWorkflowStepDto, SubmissionId, SubmissionMetadata, SubmissionType, + SubmittedNonApprovalStep, + WorkflowType, } from '../../../../../shared/types' import { PaymentNotFoundError } from '../../payments/payments.errors' import * as PaymentsService from '../../payments/payments.service' @@ -66,7 +69,10 @@ import { transformAttachmentMetasToSignedUrls, triggerVirusScanning, } from '../submission.service' -import { extractEmailConfirmationData } from '../submission.utils' +import { + buildMrfMetadata, + extractEmailConfirmationData, +} from '../submission.utils' jest.mock('src/app/services/mail/mail.service') const MockMailService = jest.mocked(MailService) @@ -1637,6 +1643,88 @@ describe('submission.service', () => { }) }) + describe('addMrfMetadata', () => { + it('should return original object without mrf metadata when submission is not multirespondent submission type', () => { + // Arrange + const mockInput = new PassThrough() + const actualTransformedData: any[] = [] + + // Act + // Build pipeline for testing + mockInput.pipe(SubmissionService.addMrfMetadata()).on('data', (data) => { + actualTransformedData.push(data) + }) + + // Emit events + const mockData = { + submissionType: SubmissionType.Encrypt, + formId: 'mockFormId', + submissionId: 'mockSubmissionId', + } + mockInput.emit('data', mockData) + mockInput.end() + + // Assert + expect(actualTransformedData).toEqual([mockData]) + }) + + it('should add mrf metadata when submission is multirespondent type', () => { + // Arrange + const WORKFLOW_STEP_1: FormWorkflowStepDto = { + _id: 'step_1_id', + workflow_type: WorkflowType.Static, + emails: ['example@example.com'], + edit: [], + } + const WORKFLOW_STEP_2: FormWorkflowStepDto = { + _id: 'step_2_id', + workflow_type: WorkflowType.Static, + emails: ['example@example.com'], + edit: [], + } + + const mockInput = new PassThrough() + const actualTransformedData: any[] = [] + + // Act + // Build pipeline for testing + mockInput.pipe(SubmissionService.addMrfMetadata()).on('data', (data) => { + actualTransformedData.push(data) + }) + + // Emit events + const mockData = { + submissionType: SubmissionType.Multirespondent, + formId: 'mockFormId', + submissionId: 'mockSubmissionId', + workflow: [WORKFLOW_STEP_1, WORKFLOW_STEP_2], + workflowStep: 0, + submittedSteps: [ + { + isApproval: false, + submittedAt: '2024-01-01T00:00:00.000Z', + } as SubmittedNonApprovalStep, + ], + } + mockInput.emit('data', mockData) + mockInput.end() + + // Assert + expect(actualTransformedData).toEqual([ + { + submissionType: SubmissionType.Multirespondent, + formId: 'mockFormId', + submissionId: 'mockSubmissionId', + mrfMeta: buildMrfMetadata({ + workflow: mockData.workflow, + workflowStep: mockData.workflowStep, + submittedSteps: mockData.submittedSteps, + }), + }, + ]) + }) + }) + describe('getSubmissionPaymentDto', () => { const MOCK_PAYMENT_ID = 'mockPaymentId' From cd734b97e822608a27d0b059d2e2b46ff219a3b2 Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Fri, 13 Dec 2024 23:32:14 +0800 Subject: [PATCH 23/35] feat: add tc for submission.server.model to fetch mrf meta --- .../__tests__/submission.server.model.spec.ts | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/src/app/models/__tests__/submission.server.model.spec.ts b/src/app/models/__tests__/submission.server.model.spec.ts index d50aa0b480..a6dbdb5b63 100644 --- a/src/app/models/__tests__/submission.server.model.spec.ts +++ b/src/app/models/__tests__/submission.server.model.spec.ts @@ -7,6 +7,7 @@ import mongoose from 'mongoose' import getSubmissionModel, { getEmailSubmissionModel, getEncryptSubmissionModel, + getMultirespondentSubmissionModel, } from 'src/app/models/submission.server.model' import { @@ -29,6 +30,7 @@ const MockDns = jest.mocked(dns) const Submission = getSubmissionModel(mongoose) const EncryptedSubmission = getEncryptSubmissionModel(mongoose) +const MultirespondentSubmission = getMultirespondentSubmissionModel(mongoose) const EmailSubmission = getEmailSubmissionModel(mongoose) const PaymentSubmission = getPaymentModel(mongoose) @@ -571,6 +573,127 @@ describe('Submission Model', () => { expect(actualResult).toEqual([]) }) }) + + describe('findSingleMetadata', () => { + it('should not return mrf metadata for storage mode form', async () => { + // Arrange + const formId = new ObjectId() + const submission = await EncryptedSubmission.create({ + form: formId, + submissionType: SubmissionType.Encrypt, + }) + + // Act + const result = await EncryptedSubmission.findSingleMetadata( + String(formId), + String(submission._id), + ) + + // Assert + expect(result?.mrf).toBeUndefined() + }) + + it('should return mrf metadata for an mrf form', async () => { + // Arrange + const formId = new ObjectId() + const workflowId = new ObjectId() + const fieldId = new ObjectId() + const submission = await MultirespondentSubmission.create({ + form: formId, + submissionType: SubmissionType.Multirespondent, + form_fields: [{ _id: fieldId, fieldType: BasicField.ShortText }], + form_logics: [], + workflow: [ + { _id: workflowId, workflow_type: WorkflowType.Static, emails: [] }, + ], + submissionPublicKey: 'test public key', + encryptedSubmissionSecretKey: 'test secret key', + encryptedContent: 'test encrypted content', + version: 1, + workflowStep: 0, + submittedSteps: [], + }) + + await submission.save() + + // Act + const result = await MultirespondentSubmission.findSingleMetadata( + String(formId), + String(submission._id), + ) + + // Assert + expect(result).toBeDefined() + expect(result?.mrf).toEqual({ + workflowStep: submission.workflowStep, + workflow: submission.workflow, + submittedSteps: submission.submittedSteps, + }) + expect(result?.refNo).toBeDefined() + expect(result?.number).toEqual(1) + }) + }) + + describe('findAllMetadataByFormId', () => { + it('should not return mrf metadata for storage mode form', async () => { + // Arrange + const formId = new ObjectId() + await EncryptedSubmission.create({ + form: formId, + submissionType: SubmissionType.Encrypt, + }) + + // Act + const result = await EncryptedSubmission.findAllMetadataByFormId( + String(formId), + ) + + // Assert + expect(result.metadata[0].mrf).toBeUndefined() + expect(result.metadata[0].refNo).toBeDefined() + expect(result.metadata[0].number).toEqual(1) + expect(result.count).toEqual(1) + }) + + it('should return mrf metadata for an mrf form', async () => { + // Arrange + const formId = new ObjectId() + const workflowId = new ObjectId() + const fieldId = new ObjectId() + const submission = await MultirespondentSubmission.create({ + form: formId, + submissionType: SubmissionType.Multirespondent, + form_fields: [{ _id: fieldId, fieldType: BasicField.ShortText }], + form_logics: [], + workflow: [ + { _id: workflowId, workflow_type: WorkflowType.Static, emails: [] }, + ], + submissionPublicKey: 'test public key', + encryptedSubmissionSecretKey: 'test secret key', + encryptedContent: 'test encrypted content', + version: 1, + workflowStep: 0, + submittedSteps: [], + }) + + await submission.save() + + // Act + const result = await MultirespondentSubmission.findAllMetadataByFormId( + String(formId), + ) + + // Assert + expect(result.count).toEqual(1) + expect(omit(result.metadata[0], ['_id', 'created'])).toEqual({ + workflowStep: submission.workflowStep, + workflow: submission.workflow, + submittedSteps: submission.submittedSteps, + }) + expect(result.metadata[0].refNo).toBeDefined() + expect(result.metadata[0].number).toEqual(1) + }) + }) }) describe('Methods', () => { From 3d80ac3f7f005b29ae5725e91ca20bd2aa164108 Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Fri, 13 Dec 2024 23:47:03 +0800 Subject: [PATCH 24/35] feat: add tc for csv generator for mrf --- .../EncryptedResponseCsvGenerator.test.ts | 60 +++++++++++++++++++ .../common/utils/mrfSubmissionView.ts | 9 ++- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.test.ts b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.test.ts index e36542e5af..d2fa5cbc91 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.test.ts +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.test.ts @@ -3,6 +3,18 @@ import { stringify } from 'csv-string' import { formatInTimeZone } from 'date-fns-tz' import { SetOptional } from 'type-fest' +import { WorkflowStatus } from '~shared/types/submission' + +import { + getCurrentStepString, + MRF_STATUS, +} from '~features/admin-form/responses/common/utils/mrfSubmissionView' +import { + MRF_CURRENT_STEP_LABEL, + MRF_FIRST_STEP_TIMESTAMP_LABEL, + MRF_STATUS_LABEL, +} from '~features/admin-form/responses/constants' + import { CsvRecordData, DecryptedSubmissionData } from '../../types' import { DisplayedResponseWithoutAnswer, @@ -361,6 +373,54 @@ describe('EncryptedResponseCsvGenerator', () => { expect(generator.hasBeenProcessed).toBe(true) }) + describe('mrf form submissions', () => { + it('should include header and data columns for mrf metadata', () => { + // Arrange + const mrfGenerator = new EncryptedResponseCsvGenerator(1, 0, true) + const mockDecryptedRecord = [generateRecord(1)] + const mockRecord = { + record: mockDecryptedRecord, + created: mockCreatedEarly, + submissionId: 'mockSubmissionId', + mrfMeta: { + workflowStatus: WorkflowStatus.REJECTED, + workflowCurrentStepNumber: 2, + workflowNumTotalSteps: 3, + }, + } + mrfGenerator.addRecord(mockRecord) + + // Act + mrfGenerator.process() + + // Assert + // Should have 1 header row and 1 submission row + expect(mrfGenerator.records.length).toEqual(2 + BOM_LENGTH) + const expectedHeaderRow = stringify([ + 'Response ID', + MRF_STATUS_LABEL, + MRF_CURRENT_STEP_LABEL, + MRF_FIRST_STEP_TIMESTAMP_LABEL, + mockDecryptedRecord[0].question, + ]) + const expectedSubmissionRow = stringify([ + mockRecord.submissionId, + MRF_STATUS.COMPLETED, + getCurrentStepString( + mockRecord.mrfMeta.workflowCurrentStepNumber, + mockRecord.mrfMeta.workflowNumTotalSteps, + ), + getFormattedDate(mockRecord.created), + mockDecryptedRecord[0].answer, + ]) + expect(mrfGenerator.records).toEqual([ + UTF8_BYTE_ORDER_MARK, + expectedHeaderRow, + expectedSubmissionRow, + ]) + }) + }) + describe('submissions with only answer key', () => { it('should handle a single submission', () => { // Arrange diff --git a/frontend/src/features/admin-form/responses/common/utils/mrfSubmissionView.ts b/frontend/src/features/admin-form/responses/common/utils/mrfSubmissionView.ts index 4b0c921a80..4530cda82a 100644 --- a/frontend/src/features/admin-form/responses/common/utils/mrfSubmissionView.ts +++ b/frontend/src/features/admin-form/responses/common/utils/mrfSubmissionView.ts @@ -1,5 +1,10 @@ import { WorkflowStatus } from '~shared/types' +export enum MRF_STATUS { + COMPLETED = 'Completed', + PENDING = 'Pending', +} + /** Gets the business friendly string for current step of workflow. */ export const getCurrentStepString = ( workflowCurrentStepNumber: number | undefined, @@ -17,9 +22,9 @@ export const getStatusFromWorkflowStatus = ( case WorkflowStatus.COMPLETED: case WorkflowStatus.APPROVED: case WorkflowStatus.REJECTED: - return 'Completed' + return MRF_STATUS.COMPLETED case WorkflowStatus.PENDING: - return 'Pending' + return MRF_STATUS.PENDING default: return '' } From 2139b9a0e5028f0d8e467dbf7cebc31447a8893f Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Fri, 13 Dec 2024 23:57:19 +0800 Subject: [PATCH 25/35] feat: fix tc missing required fields --- .../__tests__/submission.server.model.spec.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/app/models/__tests__/submission.server.model.spec.ts b/src/app/models/__tests__/submission.server.model.spec.ts index a6dbdb5b63..df7b348ae6 100644 --- a/src/app/models/__tests__/submission.server.model.spec.ts +++ b/src/app/models/__tests__/submission.server.model.spec.ts @@ -579,8 +579,13 @@ describe('Submission Model', () => { // Arrange const formId = new ObjectId() const submission = await EncryptedSubmission.create({ - form: formId, submissionType: SubmissionType.Encrypt, + form: formId, + encryptedContent: MOCK_ENCRYPTED_CONTENT, + version: 1, + authType: FormAuthType.NIL, + myInfoFields: [], + webhookResponses: [], }) // Act @@ -639,8 +644,13 @@ describe('Submission Model', () => { // Arrange const formId = new ObjectId() await EncryptedSubmission.create({ - form: formId, submissionType: SubmissionType.Encrypt, + form: formId, + encryptedContent: MOCK_ENCRYPTED_CONTENT, + version: 1, + authType: FormAuthType.NIL, + myInfoFields: [], + webhookResponses: [], }) // Act From c6eb27e64ca1a6da6ccc0a2d4996fc4ab455f646 Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Fri, 13 Dec 2024 23:57:47 +0800 Subject: [PATCH 26/35] fix: bug where approved is marked as completed instead --- src/app/modules/submission/submission.utils.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/app/modules/submission/submission.utils.ts b/src/app/modules/submission/submission.utils.ts index e5fa10b6ba..e6af3bb54f 100644 --- a/src/app/modules/submission/submission.utils.ts +++ b/src/app/modules/submission/submission.utils.ts @@ -26,6 +26,7 @@ import { SubmissionAttachment, SubmissionAttachmentsMap, SubmissionMrfMetadata, + SubmittedApprovalStep, SubmittedStep, WorkflowStatus, } from '../../../../shared/types' @@ -845,9 +846,13 @@ const getMrfSubmissionWorkflowStatus = ( return WorkflowStatus.REJECTED } if (submittedSteps.length === numTotalSteps) { + const latestApprovalStep = submittedSteps + .slice() + .reverse() + .find((step) => step.isApproval) as SubmittedApprovalStep | undefined if ( - latestSubmittedStep.isApproval && - latestSubmittedStep.status === WorkflowStatus.APPROVED + latestApprovalStep && + latestApprovalStep.status === WorkflowStatus.APPROVED ) { return WorkflowStatus.APPROVED } From 90d375b8cb86e4e6f258dadecb5a46cf3e6cd23c Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Sat, 14 Dec 2024 00:03:47 +0800 Subject: [PATCH 27/35] fix: submission.server.model tc --- .../__tests__/submission.server.model.spec.ts | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/app/models/__tests__/submission.server.model.spec.ts b/src/app/models/__tests__/submission.server.model.spec.ts index df7b348ae6..0a03aea507 100644 --- a/src/app/models/__tests__/submission.server.model.spec.ts +++ b/src/app/models/__tests__/submission.server.model.spec.ts @@ -9,6 +9,7 @@ import getSubmissionModel, { getEncryptSubmissionModel, getMultirespondentSubmissionModel, } from 'src/app/models/submission.server.model' +import { buildMrfMetadata } from 'src/app/modules/submission/submission.utils' import { BasicField, @@ -629,11 +630,13 @@ describe('Submission Model', () => { // Assert expect(result).toBeDefined() - expect(result?.mrf).toEqual({ - workflowStep: submission.workflowStep, - workflow: submission.workflow, - submittedSteps: submission.submittedSteps, - }) + expect(result?.mrf).toEqual( + buildMrfMetadata({ + workflow: submission.workflow, + workflowStep: submission.workflowStep, + submittedSteps: submission.submittedSteps, + }), + ) expect(result?.refNo).toBeDefined() expect(result?.number).toEqual(1) }) @@ -695,11 +698,13 @@ describe('Submission Model', () => { // Assert expect(result.count).toEqual(1) - expect(omit(result.metadata[0], ['_id', 'created'])).toEqual({ - workflowStep: submission.workflowStep, - workflow: submission.workflow, - submittedSteps: submission.submittedSteps, - }) + expect(result.metadata[0].mrf).toEqual( + buildMrfMetadata({ + workflowStep: submission.workflowStep, + workflow: submission.workflow, + submittedSteps: submission.submittedSteps, + }), + ) expect(result.metadata[0].refNo).toBeDefined() expect(result.metadata[0].number).toEqual(1) }) From 15f6d2f8984ec9fdae3eed604deb18eb2f4255cf Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Sat, 14 Dec 2024 00:07:46 +0800 Subject: [PATCH 28/35] fix: import ordering for lint --- .../UnlockedResponses/ResponsesTable/ResponsesTable.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx b/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx index 8b937e5b5e..11d5dc3568 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx @@ -30,12 +30,16 @@ import { centsToDollars } from '~shared/utils/payments' import Badge from '~components/Badge' import { useAdminForm } from '~features/admin-form/common/queries' +import { + MRF_CURRENT_STEP_LABEL, + MRF_FIRST_STEP_TIMESTAMP_LABEL, + MRF_STATUS_LABEL, +} from '~features/admin-form/responses/constants' import { getCurrentStepString } from '../../../../common/utils/mrfSubmissionView' import { useUnlockedResponses } from '../UnlockedResponsesProvider' import { getNetAmount } from './utils' -import { MRF_CURRENT_STEP_LABEL, MRF_STATUS_LABEL, MRF_FIRST_STEP_TIMESTAMP_LABEL } from '~features/admin-form/responses/constants' type ResponseColumnData = SubmissionMetadata From 2b9ca5d16332aaa8ed0e7cb22facf63ac5b348e4 Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Sat, 14 Dec 2024 01:22:27 +0800 Subject: [PATCH 29/35] feat: add chromatic story for mrf dashboard --- .../AdminFormResultsResponsesPage.stories.tsx | 19 ++++ .../src/mocks/msw/handlers/admin-form/form.ts | 89 ++++++++++++++++++- 2 files changed, 106 insertions(+), 2 deletions(-) diff --git a/frontend/src/features/admin-form/AdminFormResultsResponsesPage.stories.tsx b/frontend/src/features/admin-form/AdminFormResultsResponsesPage.stories.tsx index 3d095d1640..c40f0fae7d 100644 --- a/frontend/src/features/admin-form/AdminFormResultsResponsesPage.stories.tsx +++ b/frontend/src/features/admin-form/AdminFormResultsResponsesPage.stories.tsx @@ -9,6 +9,7 @@ import { createFormBuilderMocks, getAdminFormCollaborators, getAdminFormSubmissions, + getMultiRespondentSubmissionMetadataResponse, getStorageSubmissionMetadataResponse, } from '~/mocks/msw/handlers/admin-form' import { getUser } from '~/mocks/msw/handlers/user' @@ -185,6 +186,24 @@ StorageFormLoading.parameters = { ], } +export const MultiRespondentFormUnlocked = Template.bind({}) +MultiRespondentFormUnlocked.parameters = { + msw: [ + ...createFormBuilderMocks( + { + responseMode: FormResponseMode.Multirespondent, + publicKey: MOCK_KEYPAIR.publicKey, + }, + 0, + ), + getAdminFormSubmissions({ override: 5 }), + getMultiRespondentSubmissionMetadataResponse(), + getUser(), + getAdminFormCollaborators(), + ], +} +MultiRespondentFormUnlocked.play = StorageFormUnlocked.play + export const Loading = Template.bind({}) Loading.parameters = { msw: [ diff --git a/frontend/src/mocks/msw/handlers/admin-form/form.ts b/frontend/src/mocks/msw/handlers/admin-form/form.ts index b3b21a9c3c..2646c4812e 100644 --- a/frontend/src/mocks/msw/handlers/admin-form/form.ts +++ b/frontend/src/mocks/msw/handlers/admin-form/form.ts @@ -30,7 +30,10 @@ import { } from '~shared/types/form/form' import { FormLogoState } from '~shared/types/form/form_logo' import { DateString } from '~shared/types/generic' -import { SubmissionMetadataList } from '~shared/types/submission' +import { + SubmissionMetadataList, + WorkflowStatus, +} from '~shared/types/submission' import { UserDto } from '~shared/types/user' import { insertAt, reorder } from '~shared/utils/immutable-array-fns' @@ -331,6 +334,63 @@ export const MOCK_FORM_FIELDS: FormFieldDto[] = [ }, ] +const DEFAULT_MULTIRESPONDENT_METADATA = [ + [ + { + number: 1, + refNo: '62a8a7476f4f3e005bcd5ab7', + submissionTime: '14th Jun 2022, 11:20:39 pm', + mrf: { + workflowCurrentStepNumber: 1, + workflowNumTotalSteps: 5, + workflowStatus: WorkflowStatus.PENDING, + }, + }, + { + number: 2, + refNo: '62a8a7476f4f3e005bcd5ab8', + submissionTime: '14th Jun 2022, 11:21:39 pm', + mrf: { + workflowCurrentStepNumber: 3, + workflowNumTotalSteps: 3, + workflowStatus: WorkflowStatus.APPROVED, + }, + }, + { + number: 3, + refNo: '62a8a7476f4f3e005bcd5ab9', + submissionTime: '14th Jun 2022, 11:22:39 pm', + mrf: { + workflowCurrentStepNumber: 2, + workflowNumTotalSteps: 3, + workflowStatus: WorkflowStatus.REJECTED, + }, + }, + { + number: 4, + refNo: '62a8a7476f4f3e005bcd5ac0', + submissionTime: '14th Jun 2022, 11:23:39 pm', + mrf: { + workflowCurrentStepNumber: 4, + workflowNumTotalSteps: 4, + workflowStatus: WorkflowStatus.COMPLETED, + }, + }, + // simulates a submission prior to https://github.com/opengovsg/FormSG/pull/7965 + // which the workflowStatus cannot be determined as submittedSteps is undefined. + { + number: 5, + refNo: '62a8a7476f4f3e005bcd5ac1', + submissionTime: '14th Jun 2022, 11:24:39 pm', + mrf: { + workflowCurrentStepNumber: 1, + workflowNumTotalSteps: 4, + workflowStatus: undefined, + }, + }, + ], +] + const DEFAULT_STORAGE_METADATA = [ [ { @@ -784,7 +844,7 @@ export const getStorageSubmissionMetadataResponse = ( ctx.json( merge( { - count: 39, + count: DEFAULT_STORAGE_METADATA.flat().length, metadata: DEFAULT_STORAGE_METADATA[pageNum - 1], }, props, @@ -795,6 +855,31 @@ export const getStorageSubmissionMetadataResponse = ( ) } +export const getMultiRespondentSubmissionMetadataResponse = ( + props: Partial = {}, + delay: number | 'infinite' | 'real' = 0, +) => { + return rest.get( + '/api/v3/admin/forms/:formId/submissions/metadata', + (req, res, ctx) => { + const pageNum = parseInt(req.url.searchParams.get('page') ?? '1') + return res( + ctx.delay(delay), + ctx.status(200), + ctx.json( + merge( + { + count: DEFAULT_MULTIRESPONDENT_METADATA.flat().length, + metadata: DEFAULT_MULTIRESPONDENT_METADATA[pageNum - 1], + }, + props, + ), + ), + ) + }, + ) +} + export const createLogic = (delay?: number | 'infinite') => { return rest.post( '/api/v3/admin/forms/:formId/logic', From 0ef1e49b280c05c177c30fceed0acb9b3068314f Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Sun, 15 Dec 2024 19:43:22 +0800 Subject: [PATCH 30/35] chore: remove redundant comment --- .../multirespondent-submission.service.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/app/modules/submission/multirespondent-submission/multirespondent-submission.service.ts b/src/app/modules/submission/multirespondent-submission/multirespondent-submission.service.ts index 5d2f9b2080..0b083aac2b 100644 --- a/src/app/modules/submission/multirespondent-submission/multirespondent-submission.service.ts +++ b/src/app/modules/submission/multirespondent-submission/multirespondent-submission.service.ts @@ -353,7 +353,6 @@ export const createMultiRespondentFormSubmission = ({ mrfVersion, } = encryptedPayload - // For non-approval steps, we only need isApproval: false and submittedAt const submittedStepMeta: SubmittedNonApprovalStep = { isApproval: false, // first step cannot be approval step submittedAt: new Date().toISOString(), From d3004bfba8401df624a0cc48af47398631e2bc8a Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Mon, 23 Dec 2024 10:51:38 +0800 Subject: [PATCH 31/35] chore: add docs on rationale of destructuring array --- .../EncryptedResponseCsvGenerator.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts index b9823885ca..6a883d5d1b 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts @@ -118,9 +118,12 @@ export class EncryptedResponseCsvGenerator extends CsvGenerator { if (this.hasBeenProcessed) return // Create a header row in CSV using the fieldIdToQuestion map. + // NOTE: de-structuring is necessary to avoid mutating the array referenced by the `headers` array below. + // See: https://github.com/opengovsg/FormSG/pull/7965#discussion_r1883954194. const headers = this.isMrf ? [...MRF_CSV_HEADERS] : [...NON_MRF_CSV_HEADERS] this.fieldIdToQuestion.forEach((value, fieldId) => { for (let i = 0; i < this.fieldIdToNumCols[fieldId]; i++) { + // TODO: (Code quality) Refactor to avoid mutating the `headers` array. headers.push(value.question) } }) From 9ecac23cc546abfdd7ecfaa113aa1fcc009662a9 Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Mon, 23 Dec 2024 10:52:50 +0800 Subject: [PATCH 32/35] chore: rename from non-mrf to base --- .../UnlockedResponses/ResponsesTable/ResponsesTable.tsx | 6 +++--- .../EncryptedResponseCsvGenerator.ts | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx b/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx index 11d5dc3568..e31674c103 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/UnlockedResponses/ResponsesTable/ResponsesTable.tsx @@ -67,7 +67,7 @@ const CompletedBadge = () => ( ) -const NON_MRF_RESPONSE_TABLE_COLUMNS: Column[] = [ +const BASE_RESPONSE_TABLE_COLUMNS: Column[] = [ { Header: '#', accessor: 'number', @@ -204,7 +204,7 @@ const MRF_RESPONSE_TABLE_COLUMNS: Column[] = [ ] const PAYMENT_RESPONSE_TABLE_COLUMNS = - NON_MRF_RESPONSE_TABLE_COLUMNS.concat(PAYMENT_COLUMNS) + BASE_RESPONSE_TABLE_COLUMNS.concat(PAYMENT_COLUMNS) export const ResponsesTable = () => { const { data: form } = useAdminForm() @@ -245,7 +245,7 @@ export const ResponsesTable = () => { if (isPaymentsForm) { return PAYMENT_RESPONSE_TABLE_COLUMNS } - return NON_MRF_RESPONSE_TABLE_COLUMNS + return BASE_RESPONSE_TABLE_COLUMNS }, [isMultiRespondentForm, isPaymentsForm]) const { diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts index 6a883d5d1b..e7e0ad9428 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/utils/EncryptedResponseCsvGenerator/EncryptedResponseCsvGenerator.ts @@ -32,7 +32,7 @@ const MRF_CSV_HEADERS = [ MRF_FIRST_STEP_TIMESTAMP_LABEL, ] -const NON_MRF_CSV_HEADERS = ['Response ID', 'Timestamp'] +const BASE_CSV_HEADERS = ['Response ID', 'Timestamp'] export class EncryptedResponseCsvGenerator extends CsvGenerator { hasBeenProcessed: boolean @@ -120,7 +120,7 @@ export class EncryptedResponseCsvGenerator extends CsvGenerator { // Create a header row in CSV using the fieldIdToQuestion map. // NOTE: de-structuring is necessary to avoid mutating the array referenced by the `headers` array below. // See: https://github.com/opengovsg/FormSG/pull/7965#discussion_r1883954194. - const headers = this.isMrf ? [...MRF_CSV_HEADERS] : [...NON_MRF_CSV_HEADERS] + const headers = this.isMrf ? [...MRF_CSV_HEADERS] : [...BASE_CSV_HEADERS] this.fieldIdToQuestion.forEach((value, fieldId) => { for (let i = 0; i < this.fieldIdToNumCols[fieldId]; i++) { // TODO: (Code quality) Refactor to avoid mutating the `headers` array. From 1ac36204b913e48486d7dec336fc5f783fde6ae8 Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Mon, 23 Dec 2024 10:55:17 +0800 Subject: [PATCH 33/35] chore: refactor code to remove usage of undefined in argument --- src/app/models/submission.server.model.ts | 37 +++++++++++++++-------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/src/app/models/submission.server.model.ts b/src/app/models/submission.server.model.ts index 054b538629..6f110c6fb9 100644 --- a/src/app/models/submission.server.model.ts +++ b/src/app/models/submission.server.model.ts @@ -367,7 +367,11 @@ EncryptSubmissionSchema.statics.findSingleMetadata = function ( const paymentMeta = result.payments?.[0] // Build submissionMetadata object. - const metadata = buildSubmissionMetadata(result, 1, paymentMeta) + const metadata = buildSubmissionMetadata({ + result, + currentNumber: 1, + paymentMeta, + }) return metadata }) @@ -436,11 +440,11 @@ EncryptSubmissionSchema.statics.findAllMetadataByFormId = function ( const metadata = results.map((result) => { const paymentMeta = result.payments?.[0] - const metadataEntry = buildSubmissionMetadata( + const metadataEntry = buildSubmissionMetadata({ result, currentNumber, paymentMeta, - ) + }) currentNumber-- return metadataEntry @@ -592,7 +596,11 @@ MultirespondentSubmissionSchema.statics.findSingleMetadata = function ( submittedSteps: result.submittedSteps, } // Build submissionMetadata object. - const metadata = buildSubmissionMetadata(result, 1, undefined, mrfMeta) + const metadata = buildSubmissionMetadata({ + result, + currentNumber: 1, + mrfMeta, + }) return metadata }) @@ -647,12 +655,12 @@ MultirespondentSubmissionSchema.statics.findAllMetadataByFormId = function ( workflow: result.workflow, submittedSteps: result.submittedSteps, } - const metadataEntry = buildSubmissionMetadata( + const metadataEntry = buildSubmissionMetadata({ result, currentNumber, paymentMeta, mrfMeta, - ) + }) currentNumber-- return metadataEntry @@ -787,12 +795,17 @@ export const getMultirespondentSubmissionModel = ( >(SubmissionType.Multirespondent) } -const buildSubmissionMetadata = ( - result: MetadataAggregateResult, - currentNumber: number, - paymentMeta?: PaymentAggregates, - mrfMeta?: MultiRespondentAggregates, -): SubmissionMetadata => { +const buildSubmissionMetadata = ({ + result, + currentNumber, + paymentMeta, + mrfMeta, +}: { + result: MetadataAggregateResult + currentNumber: number + paymentMeta?: PaymentAggregates + mrfMeta?: MultiRespondentAggregates +}): SubmissionMetadata => { return { number: currentNumber, refNo: result._id, From 66b411ce67c9c3f3d841142128a2c29bbdf623cf Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Mon, 23 Dec 2024 10:58:47 +0800 Subject: [PATCH 34/35] feat: make typing more explicit for case in getStatusFromWorkflowStatus --- .../admin-form/responses/common/utils/mrfSubmissionView.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/features/admin-form/responses/common/utils/mrfSubmissionView.ts b/frontend/src/features/admin-form/responses/common/utils/mrfSubmissionView.ts index 4530cda82a..2543c60d08 100644 --- a/frontend/src/features/admin-form/responses/common/utils/mrfSubmissionView.ts +++ b/frontend/src/features/admin-form/responses/common/utils/mrfSubmissionView.ts @@ -17,7 +17,7 @@ export const getCurrentStepString = ( /** Gets the business friendly string for MRF submission status. */ export const getStatusFromWorkflowStatus = ( workflowStatus: WorkflowStatus | undefined, -) => { +): MRF_STATUS | '' => { switch (workflowStatus) { case WorkflowStatus.COMPLETED: case WorkflowStatus.APPROVED: @@ -25,7 +25,7 @@ export const getStatusFromWorkflowStatus = ( return MRF_STATUS.COMPLETED case WorkflowStatus.PENDING: return MRF_STATUS.PENDING - default: + case undefined: return '' } } From 2b8d32dfc0206b8e50dd1db796b70b843e1f3420 Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Mon, 23 Dec 2024 13:21:46 +0800 Subject: [PATCH 35/35] fix: remove global access in decrypt and config in vite --- .../ResponsesPage/storage/worker/decryption.worker.ts | 11 +---------- frontend/vite.config.ts | 5 ----- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/frontend/src/features/admin-form/responses/ResponsesPage/storage/worker/decryption.worker.ts b/frontend/src/features/admin-form/responses/ResponsesPage/storage/worker/decryption.worker.ts index 6ef4060bb1..3fb7ac1730 100644 --- a/frontend/src/features/admin-form/responses/ResponsesPage/storage/worker/decryption.worker.ts +++ b/frontend/src/features/admin-form/responses/ResponsesPage/storage/worker/decryption.worker.ts @@ -13,15 +13,6 @@ import { } from '../types' import { CsvRecord } from '../utils/CsvRecord.class' -// Fixes issue raised at https://stackoverflow.com/questions/66472945/referenceerror-refreshreg-is-not-defined -// Something to do with babel-loader. -if (import.meta.env.NODE_ENV !== 'production') { - // eslint-disable-next-line - ;(global as any).$RefreshReg$ = () => {} - // eslint-disable-next-line - ;(global as any).$RefreshSig$ = () => () => {} -} - const queue = new PQueue({ concurrency: 1 }) /** @@ -212,7 +203,7 @@ async function decryptIntoCsv( } catch (error) { csvRecord.setStatus(CsvRecordStatus.Error, 'Decryption Error') } - } catch (err) { + } catch (error) { csvRecord = new CsvRecord( CsvRecordStatus.Error, formatInTimeZone(new Date(), 'Asia/Singapore', 'dd MMM yyyy hh:mm:ss z'), diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index d6476bcfd4..c4277dc150 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -61,10 +61,5 @@ export default defineConfig(() => { plugins: () => [tsconfigPaths()], format: 'es' as const, }, - define: { - // On local dev, global is undefined, causing decryption workers to fail. - // This is a workaround based on https://github.com/vitejs/vite/discussions/5912. - global: 'globalThis', - }, } })