diff --git a/apps/backend/src/app/modules/submission/__tests__/submission.service.spec.ts b/apps/backend/src/app/modules/submission/__tests__/submission.service.spec.ts index 676ee83fdc..25a73cf44d 100644 --- a/apps/backend/src/app/modules/submission/__tests__/submission.service.spec.ts +++ b/apps/backend/src/app/modules/submission/__tests__/submission.service.spec.ts @@ -49,6 +49,7 @@ import { StorageModeSubmissionCursorData, StorageModeSubmissionData, } from 'src/types' +import { ParsedClearAttachmentFieldResponseV4 } from 'src/types/api' import { PaymentNotFoundError } from '../../payments/payments.errors' import * as PaymentsService from '../../payments/payments.service' @@ -71,6 +72,7 @@ import { getQuarantinePresignedPostData, transformAttachmentMetasToSignedUrls, triggerGuardDutyScanning, + triggerGuardDutyScanThenDownloadCleanFileChainV4, } from '../submission.service' import { buildMrfMetadata, @@ -3034,6 +3036,82 @@ describe('submission.service', () => { expect(actualResult._unsafeUnwrap().toString()).toEqual(content) }) }) + + describe('triggerGuardDutyScanThenDownloadCleanFileChainV4', () => { + // Regression guard for the MRF V4 wiring: after a clean scan the real + // uploaded filename must land in `answer.value`, not just `answer.filename`. + // Downstream sinks — email/PDF Q&A, response JSON, the V4->V3 webhook + // adapter, the encrypted-at-rest answer, and the admin CSV/attachment + // download name — all read `answer.value`. Previously it stayed as the + // quarantine bucket key (a bare UUID), which dropped the filename and its + // extension. This mirrors the V3 chain's `answer: response.filename`. + const MOCK_QUARANTINE_KEY = '1b90195b-ce8a-4590-810b-04ebaef8e4dd' + const MOCK_CLEAN_FILE_KEY = '0f3d2e22-d2aa-44f8-965a-27e46102936e' + const MOCK_FILENAME = 'my report.pdf' + const MOCK_CLEAN_CONTENT = 'clean file contents' + + afterEach(() => jest.restoreAllMocks()) + + const makeV4AttachmentResponse = () => + ({ + fieldType: BasicField.Attachment, + question: 'Upload file', + answer: { + value: MOCK_QUARANTINE_KEY, + hasBeenScanned: false, + filename: MOCK_FILENAME, + content: Buffer.from(''), + }, + }) as unknown as ParsedClearAttachmentFieldResponseV4 + + const mockCleanScanAndDownload = () => { + // Lambda returns a clean-bucket key (must be a valid UUID for downloadCleanFile). + jest.spyOn(aws.guarddutyLambda, 'invoke').mockImplementationOnce(() => { + return Promise.resolve({ + Payload: JSON.stringify({ + statusCode: 200, + body: JSON.stringify({ + cleanFileKey: MOCK_CLEAN_FILE_KEY, + destinationVersionId: 'version-id', + }), + }), + }) + }) + // S3 streams back the clean file content. + const mockGetObject = jest.fn().mockReturnValue({ + createReadStream: () => + new Readable({ + read() { + this.push(MOCK_CLEAN_CONTENT) + this.push(null) + }, + }), + }) + jest.spyOn(aws.s3, 'getObject').mockImplementationOnce(mockGetObject) + } + + it('should promote the real filename (with extension) into answer.value after a clean scan', async () => { + // Arrange + mockCleanScanAndDownload() + + // Act + const actualResult = + await triggerGuardDutyScanThenDownloadCleanFileChainV4( + makeV4AttachmentResponse(), + MOCK_FORM_ID, + ) + + // Assert + expect(actualResult.isOk()).toEqual(true) + const scanned = actualResult._unsafeUnwrap() + // The canonical, durable answer field must carry the filename — not the + // quarantine bucket key that was used only to trigger the scan. + expect(scanned.answer.value).toEqual(MOCK_FILENAME) + expect(scanned.answer.value).not.toEqual(MOCK_QUARANTINE_KEY) + expect(scanned.answer.filename).toEqual(MOCK_FILENAME) + expect(scanned.answer.content.toString()).toEqual(MOCK_CLEAN_CONTENT) + }) + }) }) /** diff --git a/apps/backend/src/app/modules/submission/multirespondent-submission/__tests__/multirespondent-submission.middleware.spec.ts b/apps/backend/src/app/modules/submission/multirespondent-submission/__tests__/multirespondent-submission.middleware.spec.ts index 3789d9e0e1..1805a5400c 100644 --- a/apps/backend/src/app/modules/submission/multirespondent-submission/__tests__/multirespondent-submission.middleware.spec.ts +++ b/apps/backend/src/app/modules/submission/multirespondent-submission/__tests__/multirespondent-submission.middleware.spec.ts @@ -694,12 +694,14 @@ describe('Multirespondent Submission Middleware', () => { responses: {}, }) - jest.mocked(SpcpUtil.createNdiResponsesV3FromRecord).mockReturnValue({ + jest.mocked(SpcpUtil.createNdiResponsesV4FromRecord).mockReturnValue({ 'SingPass Validated NRIC': { fieldType: BasicField.Nric, - answer: 'S9812379B', + answer: { value: 'S9812379B' }, + question: 'SingPass Validated NRIC', + provenance: {}, }, - }) + } as any) const mockNext = jest.fn() @@ -722,7 +724,7 @@ describe('Multirespondent Submission Middleware', () => { // Assert expect( - jest.mocked(SpcpUtil.createNdiResponsesV3FromRecord), + jest.mocked(SpcpUtil.createNdiResponsesV4FromRecord), ).toHaveBeenCalled() expect(mockReq.formsg.encryptedPayload.responses).toHaveProperty( 'SingPass Validated NRIC', @@ -774,12 +776,14 @@ describe('Multirespondent Submission Middleware', () => { }, }) - jest.mocked(SpcpUtil.createNdiResponsesV3FromRecord).mockReturnValue({ + jest.mocked(SpcpUtil.createNdiResponsesV4FromRecord).mockReturnValue({ 'SingPass Validated NRIC': { fieldType: BasicField.Nric, - answer: 'S1234567A', + answer: { value: 'S1234567A' }, + question: 'SingPass Validated NRIC', + provenance: {}, }, - }) + } as any) jest .mocked(VerifiedContentService.getVerifiedContent) @@ -845,7 +849,7 @@ describe('Multirespondent Submission Middleware', () => { describe('encryptSubmission', () => { const MOCK_FORM_ID = new ObjectId().toHexString() - const MOCK_FORM = { + const MOCK_FORM_BASE = { _id: MOCK_FORM_ID, publicKey: 'mockPublicKey', form_fields: [ @@ -854,10 +858,15 @@ describe('Multirespondent Submission Middleware', () => { } as any const MOCK_RESPONSES = { - field1: { fieldType: BasicField.ShortText, answer: 'hello' }, + field1: { + fieldType: BasicField.ShortText, + answer: 'hello', + question: 'Field 1', + provenance: {}, + }, } - const createMockEncryptReq = (isV4FlagOn: boolean) => + const createMockEncryptReq = (hasWebhook: boolean) => ({ params: { formId: MOCK_FORM_ID }, body: { @@ -867,10 +876,15 @@ describe('Multirespondent Submission Middleware', () => { responseMetadata: {}, }, formsg: { - formDef: MOCK_FORM, + formDef: hasWebhook + ? { + ...MOCK_FORM_BASE, + webhook: { url: 'https://example.com/webhook' }, + } + : MOCK_FORM_BASE, }, growthbook: { - isOn: jest.fn().mockReturnValue(isV4FlagOn), + isOn: jest.fn().mockReturnValue(false), setAttributes: jest.fn().mockResolvedValue(undefined), getAttributes: jest.fn().mockReturnValue({}), }, @@ -905,21 +919,9 @@ describe('Multirespondent Submission Middleware', () => { mockDecrypt.mockReturnValue({ responses: MOCK_RESPONSES }) }) - it('should encrypt responses as V3 and set mrfVersion to 1 when feature flag is off', async () => { - const mockReq = createMockEncryptReq(false) - const mockNext = jest.fn() - const mockRes = createMockRes() - - await encryptSubmission(mockReq, mockRes as any, mockNext) - - expect(jest.mocked(adaptV3ToV4)).not.toHaveBeenCalled() - expect(mockReq.formsg.encryptedPayload.mrfVersion).toBe(1) - expect(mockNext).toHaveBeenCalled() - }) - - it('should adapt V3 responses to V4 format and set mrfVersion to 2 when feature flag is on', async () => { - jest.mocked(adaptV3ToV4).mockReturnValue({ - field1: { answer: 'hello', question: 'Field 1', provenance: {} }, + it('should encrypt responses as V3 and set mrfVersion to 1 when form has a webhook url', async () => { + jest.mocked(adaptV4ToV3).mockReturnValue({ + field1: { fieldType: BasicField.ShortText, answer: 'hello' }, } as any) const mockReq = createMockEncryptReq(true) @@ -928,11 +930,8 @@ describe('Multirespondent Submission Middleware', () => { await encryptSubmission(mockReq, mockRes as any, mockNext) - expect(jest.mocked(adaptV3ToV4)).toHaveBeenCalledWith( - { field1: { fieldType: BasicField.ShortText, answer: 'hello' } }, - { formFields: { field1: { question: 'Field 1' } }, provenance: {} }, - ) - expect(mockReq.formsg.encryptedPayload.mrfVersion).toBe(2) + expect(jest.mocked(adaptV4ToV3)).toHaveBeenCalledWith(MOCK_RESPONSES) + expect(mockReq.formsg.encryptedPayload.mrfVersion).toBe(1) expect(mockNext).toHaveBeenCalled() }) @@ -980,13 +979,13 @@ describe('Multirespondent Submission Middleware', () => { { step: 1, edit: [EDITABLE_FIELD_ID] }, // only field1 editable at step 1 ] - // mrfVersion: 2 means responses were encrypted in V4 format + // mrfVersion: 1 means previous submission was encrypted in V3 format // workflowStep: 0 means the current incoming submission is at step 1 - const MOCK_MRF_SUBMISSION_V2 = { + const MOCK_MRF_SUBMISSION_V1 = { form: MOCK_FORM_ID, - encryptedContent: 'v4-encrypted-content', + encryptedContent: 'v3-encrypted-content', version: 1, - mrfVersion: 2, + mrfVersion: 1, form_fields: SNAPSHOT_FORM_FIELDS, form_logics: [], workflow: SNAPSHOT_WORKFLOW, @@ -999,29 +998,31 @@ describe('Multirespondent Submission Middleware', () => { getWebhookView: jest.fn(), } as any - // V4 decrypted responses (each entry has provenance) - const MOCK_V4_DECRYPTED_RESPONSES = { + // V3 decrypted responses returned by decryptFromSubmissionKey for a V3-encrypted previous submission + const MOCK_V3_DECRYPTED_RESPONSES = { [EDITABLE_FIELD_ID]: { + fieldType: BasicField.ShortText, answer: 'original', - question: 'Editable Field', - provenance: {}, }, [NON_EDITABLE_FIELD_ID]: { + fieldType: BasicField.ShortText, answer: 'locked-value', - question: 'Non-editable Field', - provenance: {}, }, } - // V3 responses produced by adaptV4ToV3 - const MOCK_V3_CONVERTED_RESPONSES = { + // V4 responses produced by adaptV3ToV4 (V4 shape with provenance) + const MOCK_V4_ADAPTED_RESPONSES = { [EDITABLE_FIELD_ID]: { fieldType: BasicField.ShortText, - answer: 'original', + answer: { value: 'original' }, + question: 'Editable Field', + provenance: {}, }, [NON_EDITABLE_FIELD_ID]: { fieldType: BasicField.ShortText, - answer: 'locked-value', + answer: { value: 'locked-value' }, + question: 'Non-editable Field', + provenance: {}, }, } @@ -1036,16 +1037,21 @@ describe('Multirespondent Submission Middleware', () => { ;( formsgSdk.cryptoV3.decryptFromSubmissionKey as jest.Mock ).mockReturnValue({ - responses: MOCK_V4_DECRYPTED_RESPONSES, + responses: MOCK_V3_DECRYPTED_RESPONSES, verified: {}, submissionSecretKey: '', }) - jest.mocked(isFieldResponsesV4).mockReturnValue(true) + // Previous decrypted responses are V3-shaped, so isFieldResponsesV4 must return false + // to trigger the V3->V4 adaptation path + jest.mocked(isFieldResponsesV4).mockReturnValue(false) + jest.mocked(adaptV3ToV4).mockReturnValue(MOCK_V4_ADAPTED_RESPONSES as any) + + // adaptV4ToV3 is still called once on req.body.responses for logic evaluation jest .mocked(adaptV4ToV3) - .mockReturnValue(MOCK_V3_CONVERTED_RESPONSES as any) + .mockReturnValue(MOCK_V3_DECRYPTED_RESPONSES as any) jest .mocked(LogicAdaptor.getVisibleFieldIdsV3) @@ -1057,10 +1063,10 @@ describe('Multirespondent Submission Middleware', () => { jest .mocked(MrfUtils.validateMrfFieldResponses) - .mockReturnValue(ok(MOCK_V3_CONVERTED_RESPONSES) as any) + .mockReturnValue(ok(MOCK_V4_ADAPTED_RESPONSES) as any) }) - it('should call adaptV4ToV3 and call next when previous mrfVersion is 2 and non-editable fields match', async () => { + it('should call adaptV3ToV4 and call next when previous mrfVersion is 1 and non-editable fields match', async () => { const mockReq = createMockReq({ formId: MOCK_FORM_ID, submissionId: MOCK_SUBMISSION_ID, @@ -1068,11 +1074,15 @@ describe('Multirespondent Submission Middleware', () => { mockReq.body.responses = { [EDITABLE_FIELD_ID]: { fieldType: BasicField.ShortText, - answer: 'updated', + answer: { value: 'updated' }, + question: 'Editable Field', + provenance: {}, }, [NON_EDITABLE_FIELD_ID]: { fieldType: BasicField.ShortText, - answer: 'locked-value', + answer: { value: 'locked-value' }, + question: 'Non-editable Field', + provenance: {}, }, } mockReq.body.submissionSecretKey = 'submission-secret-key' @@ -1083,7 +1093,7 @@ describe('Multirespondent Submission Middleware', () => { form_logics: [], workflow: SNAPSHOT_WORKFLOW, }, - mrfSubmission: MOCK_MRF_SUBMISSION_V2, + mrfSubmission: MOCK_MRF_SUBMISSION_V1, } const mockNext = jest.fn() @@ -1091,24 +1101,14 @@ describe('Multirespondent Submission Middleware', () => { await validateMultirespondentSubmission(mockReq, mockRes as any, mockNext) - expect(jest.mocked(adaptV4ToV3)).toHaveBeenCalledWith( - MOCK_V4_DECRYPTED_RESPONSES, + expect(jest.mocked(adaptV3ToV4)).toHaveBeenCalledWith( + MOCK_V3_DECRYPTED_RESPONSES, + { formFields: {}, provenance: {} }, ) expect(mockNext).toHaveBeenCalled() }) - it('should reject submission when a non-editable field is tampered after V4-to-V3 conversion', async () => { - jest.mocked(adaptV4ToV3).mockReturnValue({ - [EDITABLE_FIELD_ID]: { - fieldType: BasicField.ShortText, - answer: 'original', - }, - [NON_EDITABLE_FIELD_ID]: { - fieldType: BasicField.ShortText, - answer: 'locked-value', - }, - } as any) - + it('should reject submission when a non-editable field is tampered after V3-to-V4 conversion', async () => { const mockReq = createMockReq({ formId: MOCK_FORM_ID, submissionId: MOCK_SUBMISSION_ID, @@ -1116,11 +1116,15 @@ describe('Multirespondent Submission Middleware', () => { mockReq.body.responses = { [EDITABLE_FIELD_ID]: { fieldType: BasicField.ShortText, - answer: 'updated', + answer: { value: 'updated' }, + question: 'Editable Field', + provenance: {}, }, [NON_EDITABLE_FIELD_ID]: { fieldType: BasicField.ShortText, - answer: 'tampered', // differs from 'locked-value' + answer: { value: 'tampered' }, // differs from 'locked-value' + question: 'Non-editable Field', + provenance: {}, }, } mockReq.body.submissionSecretKey = 'submission-secret-key' @@ -1131,7 +1135,7 @@ describe('Multirespondent Submission Middleware', () => { form_logics: [], workflow: SNAPSHOT_WORKFLOW, }, - mrfSubmission: MOCK_MRF_SUBMISSION_V2, + mrfSubmission: MOCK_MRF_SUBMISSION_V1, } const mockNext = jest.fn() @@ -1139,8 +1143,9 @@ describe('Multirespondent Submission Middleware', () => { await validateMultirespondentSubmission(mockReq, mockRes as any, mockNext) - expect(jest.mocked(adaptV4ToV3)).toHaveBeenCalledWith( - MOCK_V4_DECRYPTED_RESPONSES, + expect(jest.mocked(adaptV3ToV4)).toHaveBeenCalledWith( + MOCK_V3_DECRYPTED_RESPONSES, + { formFields: {}, provenance: {} }, ) expect(mockNext).not.toHaveBeenCalled() expect(mockRes.status).toHaveBeenCalledWith(400) diff --git a/apps/backend/src/app/modules/submission/multirespondent-submission/__tests__/multirespondent-submission.service.spec.ts b/apps/backend/src/app/modules/submission/multirespondent-submission/__tests__/multirespondent-submission.service.spec.ts index e3a8828778..290fc4bf6d 100644 --- a/apps/backend/src/app/modules/submission/multirespondent-submission/__tests__/multirespondent-submission.service.spec.ts +++ b/apps/backend/src/app/modules/submission/multirespondent-submission/__tests__/multirespondent-submission.service.spec.ts @@ -1509,11 +1509,11 @@ describe('multirespondent-submission.service', () => { }, [yesNoFieldId1]: { fieldType: BasicField.YesNo, - answer: 'Yes', + answer: { value: 'Yes' }, }, [yesNoFieldId2]: { fieldType: BasicField.YesNo, - answer: 'No', + answer: { value: 'No' }, }, } as FieldResponsesV3 @@ -1648,11 +1648,11 @@ describe('multirespondent-submission.service', () => { }, [yesNoFieldId1]: { fieldType: BasicField.YesNo, - answer: 'Yes', + answer: { value: 'Yes' }, }, [yesNoFieldId2]: { fieldType: BasicField.YesNo, - answer: 'No', + answer: { value: 'No' }, }, } as FieldResponsesV3 @@ -1801,11 +1801,11 @@ describe('multirespondent-submission.service', () => { }, [yesNoFieldId1]: { fieldType: BasicField.YesNo, - answer: 'Yes', + answer: { value: 'Yes' }, }, [yesNoFieldId2]: { fieldType: BasicField.YesNo, - answer: 'No', + answer: { value: 'No' }, }, } as FieldResponsesV3 @@ -1948,11 +1948,11 @@ describe('multirespondent-submission.service', () => { }, [yesNoFieldId1]: { fieldType: BasicField.YesNo, - answer: 'Yes', + answer: { value: 'Yes' }, }, [yesNoFieldId2]: { fieldType: BasicField.YesNo, - answer: 'Yes', + answer: { value: 'Yes' }, }, } @@ -2094,7 +2094,7 @@ describe('multirespondent-submission.service', () => { }, [yesNoFieldId1]: { fieldType: BasicField.YesNo, - answer: 'No', + answer: { value: 'No' }, }, } as FieldResponsesV3 @@ -2244,7 +2244,7 @@ describe('multirespondent-submission.service', () => { }, [yesNoFieldId1]: { fieldType: BasicField.YesNo, - answer: 'No', + answer: { value: 'No' }, }, } as FieldResponsesV3 @@ -2399,11 +2399,11 @@ describe('multirespondent-submission.service', () => { }, [yesNoFieldId1]: { fieldType: BasicField.YesNo, - answer: 'Yes', + answer: { value: 'Yes' }, }, [yesNoFieldId2]: { fieldType: BasicField.YesNo, - answer: 'No', + answer: { value: 'No' }, }, } diff --git a/apps/backend/src/app/modules/submission/multirespondent-submission/__tests__/multirespondent-submission.utils.spec.ts b/apps/backend/src/app/modules/submission/multirespondent-submission/__tests__/multirespondent-submission.utils.spec.ts index 0fd0328362..d833ee90da 100644 --- a/apps/backend/src/app/modules/submission/multirespondent-submission/__tests__/multirespondent-submission.utils.spec.ts +++ b/apps/backend/src/app/modules/submission/multirespondent-submission/__tests__/multirespondent-submission.utils.spec.ts @@ -2,24 +2,16 @@ import { generateDefaultField } from '__tests__/unit/backend/helpers/generate-fo import { ObjectId } from 'bson' import { CLIENT_CHECKBOX_OTHERS_INPUT_VALUE } from 'formsg-shared/constants/form' import { - AddressAttributes, - AddressResponseV3, - AttachmentResponseV3, BasicField, - CheckboxResponseV3, ChildBirthRecordsResponseV3, - EmailResponseV3, FieldResponsesV3, FormFieldDto, FormWorkflowStepConditional, FormWorkflowStepDto, LongTextResponseV3, - NumberResponseV3, ShortTextResponseV3, - SignatureFieldResponseV3, SignatureVectorArray, SubmissionType, - TableResponseV3, WorkflowStatus, WorkflowType, } from 'formsg-shared/types' @@ -41,9 +33,10 @@ import { ITableFieldSchema, MultirespondentSubmissionData, } from 'src/types' +import { ParsedClearFormFieldResponsesV4 } from 'src/types/api' import * as fieldValidation from '../../../../utils/field-validation' -import { ValidateFieldErrorV3 } from '../../submission.errors' +import { ValidateFieldErrorV4 } from '../../submission.errors' import { buildMrfResponseJson, createMultirespondentSubmissionDto, @@ -396,16 +389,16 @@ describe('multirespondent-submission.utils', () => { // Assert expect(result.isErr()).toBe(true) - expect(result._unsafeUnwrapErr()).toBeInstanceOf(ValidateFieldErrorV3) + expect(result._unsafeUnwrapErr()).toBeInstanceOf(ValidateFieldErrorV4) expect(result._unsafeUnwrapErr().message).toBe( 'Children field type is not supported for MRF submisisons', ) }) - it('should invoke validateFieldV3 with isVisible true when non-hidden and supported field type is submitted', () => { + it('should invoke validateFieldV4 with isVisible true when non-hidden and supported field type is submitted', () => { // Arrange - const validateFieldV3Mock = jest - .spyOn(fieldValidation, 'validateFieldV3') + const validateFieldV4Mock = jest + .spyOn(fieldValidation, 'validateFieldV4') .mockReturnValue(ok(true)) const mockFormId = 'mockFormId' const field1Id = 'field1' @@ -429,20 +422,20 @@ describe('multirespondent-submission.utils', () => { }) // Assert - expect(validateFieldV3Mock).toHaveBeenCalledWith({ + expect(validateFieldV4Mock).toHaveBeenCalledWith({ formId: mockFormId, formField: mockFormFields[0], response: mockResponses.field1, isVisible: true, }) - expect(validateFieldV3Mock).toHaveBeenCalledOnce() + expect(validateFieldV4Mock).toHaveBeenCalledOnce() }) - it('should invoke validateFieldV3 with isVisible false when hidden and supported field type is submitted', () => { + it('should invoke validateFieldV4 with isVisible false when hidden and supported field type is submitted', () => { // Arrange - const validateFieldV3Mock = jest - .spyOn(fieldValidation, 'validateFieldV3') + const validateFieldV4Mock = jest + .spyOn(fieldValidation, 'validateFieldV4') .mockReturnValue(ok(true)) const mockFormId = 'mockFormId' const field1Id = 'field1' @@ -472,21 +465,76 @@ describe('multirespondent-submission.utils', () => { }) // Assert - expect(validateFieldV3Mock).toHaveBeenCalledWith({ + expect(validateFieldV4Mock).toHaveBeenCalledWith({ formId: mockFormId, formField: mockFormFields[0], response: mockResponses.field1, isVisible: false, }) - expect(validateFieldV3Mock).toHaveBeenCalledWith({ + expect(validateFieldV4Mock).toHaveBeenCalledWith({ formId: mockFormId, formField: mockFormFields[1], response: mockResponses.field2, isVisible: true, }) - expect(validateFieldV3Mock).toHaveBeenCalledTimes(2) + expect(validateFieldV4Mock).toHaveBeenCalledTimes(2) + }) + + it('should thread previousResponses through to validateFieldV4 as prevResponse', () => { + // Regression test: the V4 migration dropped previousResponses, causing + // carried-forward verifiable fields (verified in an earlier MRF step, + // whose OTP signature has since expired) to be re-authenticated and + // rejected. Threading prevResponse lets checkIsResponseChangedV4 skip + // fields the current respondent did not change. + const validateFieldV4Mock = jest + .spyOn(fieldValidation, 'validateFieldV4') + .mockReturnValue(ok(true)) + const mockFormId = 'mockFormId' + const emailFieldId = 'emailField' + const mockVisibleFieldIds = new Set([emailFieldId]) + const mockFormFields = [ + generateDefaultField(BasicField.Email, { _id: emailFieldId }), + ] + const carriedForwardAnswer = { + value: 'alice@example.com', + signature: 'stale-but-unchanged-signature', + } + const mockResponses = { + [emailFieldId]: { + fieldType: BasicField.Email, + question: 'Email', + answer: carriedForwardAnswer, + provenance: {}, + }, + } as unknown as ParsedClearFormFieldResponsesV4 + const mockPreviousResponses = { + [emailFieldId]: { + fieldType: BasicField.Email, + question: 'Email', + answer: carriedForwardAnswer, + provenance: {}, + }, + } as unknown as ParsedClearFormFieldResponsesV4 + + // Act + validateMrfFieldResponses({ + formId: mockFormId, + visibleFieldIds: mockVisibleFieldIds, + formFields: mockFormFields as FormFieldDto[], + responses: mockResponses, + previousResponses: mockPreviousResponses, + }) + + // Assert + expect(validateFieldV4Mock).toHaveBeenCalledWith({ + formId: mockFormId, + formField: mockFormFields[0], + response: mockResponses[emailFieldId], + prevResponse: mockPreviousResponses[emailFieldId], + isVisible: true, + }) }) }) @@ -509,17 +557,26 @@ describe('multirespondent-submission.utils', () => { fieldType: BasicField.Email, } as IEmailFieldSchema, ] - const responses: FieldResponsesV3 = { + const responses = { '1': { fieldType: BasicField.ShortText, - answer: 'Test answer', - } as ShortTextResponseV3, - '2': { fieldType: BasicField.Number, answer: '42' } as NumberResponseV3, + answer: { value: 'Test answer' }, + question: 'Short Text', + provenance: {}, + }, + '2': { + fieldType: BasicField.Number, + answer: { value: '42' }, + question: 'Number', + provenance: {}, + }, '3': { fieldType: BasicField.Email, answer: { value: 'test@example.com' }, - } as EmailResponseV3, - } + question: 'Email', + provenance: {}, + }, + } as any const result = getQuestionAnswerPairsForMultipleFields({ formFields, @@ -549,12 +606,14 @@ describe('multirespondent-submission.utils', () => { fieldType: BasicField.Attachment, } as IAttachmentFieldSchema, ] - const responses: FieldResponsesV3 = { + const responses = { '1': { fieldType: BasicField.Attachment, - answer: { answer: 'file.pdf' }, - } as AttachmentResponseV3, - } + answer: { value: 'file.pdf', hasBeenScanned: true }, + question: 'File Upload', + provenance: {}, + }, + } as any const result = getQuestionAnswerPairsForMultipleFields({ formFields, @@ -591,22 +650,26 @@ describe('multirespondent-submission.utils', () => { ], } as ITableFieldSchema, ] - const responses: FieldResponsesV3 = { + const responses = { '1': { fieldType: BasicField.Table, - answer: [ - { col1: 'Alice', col2: '30' }, - { col1: 'Bob', col2: '25' }, - ], - } as TableResponseV3, + answer: { + row0: { rowNum: 0, value: { col1: 'Alice', col2: '30' } }, + row1: { rowNum: 1, value: { col1: 'Bob', col2: '25' } }, + }, + question: 'Table of Name and Age', + provenance: {}, + }, '2': { fieldType: BasicField.Table, - answer: [ - { col3: 'Swimming', col4: '5' }, - { col3: 'Reading', col4: '10' }, - ], - } as TableResponseV3, - } + answer: { + row0: { rowNum: 0, value: { col3: 'Swimming', col4: '5' } }, + row1: { rowNum: 1, value: { col3: 'Reading', col4: '10' } }, + }, + question: 'Table of Hobbies', + provenance: {}, + }, + } as any const result = getQuestionAnswerPairsForMultipleFields({ formFields, @@ -645,15 +708,17 @@ describe('multirespondent-submission.utils', () => { fieldType: BasicField.Checkbox, } as ICheckboxFieldSchema, ] - const responses: FieldResponsesV3 = { + const responses = { '1': { fieldType: BasicField.Checkbox, answer: { value: ['Option 1', 'Option 2', CLIENT_CHECKBOX_OTHERS_INPUT_VALUE], othersInput: 'Custom Option', }, - } as CheckboxResponseV3, - } + question: 'Checkbox', + provenance: {}, + }, + } as any const result = getQuestionAnswerPairsForMultipleFields({ formFields, @@ -663,7 +728,7 @@ describe('multirespondent-submission.utils', () => { expect(result).toEqual([ { question: 'Checkbox', - answer: 'Option 1, Option 2, Others: Custom Option', + answer: 'Option 1, Option 2, Custom Option', fieldType: BasicField.Checkbox, }, ]) @@ -677,21 +742,21 @@ describe('multirespondent-submission.utils', () => { fieldType: BasicField.Address, } as IAddressCompoundFieldSchema, ] - const responses: FieldResponsesV3 = { + const responses = { '1': { fieldType: BasicField.Address, answer: { - addressSubFields: { - postalCode: '650161', - blockNumber: '161', - streetName: 'BUKIT BATOK STREET 11', - buildingName: '', - levelNumber: '1', - unitNumber: '1', - } as AddressAttributes, + postalCode: { value: '650161' }, + blockNumber: { value: '161' }, + streetName: { value: 'BUKIT BATOK STREET 11' }, + buildingName: { value: '' }, + levelNumber: { value: '1' }, + unitNumber: { value: '1' }, }, - } as AddressResponseV3, - } + question: 'Address', + provenance: {}, + }, + } as any const result = getQuestionAnswerPairsForMultipleFields({ formFields, @@ -721,15 +786,17 @@ describe('multirespondent-submission.utils', () => { [[40, 40, 0.5]], ] - const responses: FieldResponsesV3 = { + const responses = { '1': { fieldType: BasicField.Signature, answer: { type: 'draw', value: MOCK_SIGNATURE_VALUE, - } as SignatureFieldResponseV3, + }, + question: 'Signature', + provenance: {}, }, - } + } as any const result = getQuestionAnswerPairsForMultipleFields({ formFields, @@ -742,7 +809,7 @@ describe('multirespondent-submission.utils', () => { expect(result).toEqual([ { - question: '[Signature] Signature', + question: '[signature] Signature', answer: 'Signature captured', fieldType: BasicField.Signature, signatureDataPngDataUri: expectedSignatureDataPngDataUri, @@ -759,15 +826,17 @@ describe('multirespondent-submission.utils', () => { } as ISignatureFieldSchema, ] - const responses: FieldResponsesV3 = { + const responses = { '1': { fieldType: BasicField.Signature, answer: { type: 'draw', value: [[[10, 20, 0.5]], [[40, 40, 0.5]]], - } as SignatureFieldResponseV3, + }, + question: 'Signature', + provenance: {}, }, - } + } as any const result = getQuestionAnswerPairsForMultipleFields({ formFields, @@ -776,7 +845,7 @@ describe('multirespondent-submission.utils', () => { expect(result).toEqual([ { - question: '[Signature] Signature', + question: '[signature] Signature', answer: 'Signature captured', fieldType: BasicField.Signature, signatureDataPngDataUri: undefined, @@ -786,12 +855,14 @@ describe('multirespondent-submission.utils', () => { it('should handle Ndi fields correctly', () => { const formFields: FormFieldSchema[] = [] - const responses: FieldResponsesV3 = { + const responses = { 'SingPass Validated NRIC': { fieldType: BasicField.Nric, - answer: 'S1234567A', + answer: { value: 'S1234567A' }, + question: 'SingPass Validated NRIC', + provenance: {}, }, - } + } as any const result = getQuestionAnswerPairsForMultipleFields({ formFields, @@ -834,13 +905,17 @@ describe('multirespondent-submission.utils', () => { const mockResponsesA = { [mockConditionalFieldId]: { fieldType: BasicField.Dropdown, - answer: 'Option A', + answer: { value: 'Option A' }, + question: 'Dropdown', + provenance: {}, }, [mockShortTextFieldId]: { fieldType: BasicField.ShortText, - answer: 'Some text response', + answer: { value: 'Some text response' }, + question: 'Short Text', + provenance: {}, }, - } as FieldResponsesV3 + } as any const mockWorkflowStep = { workflow_type: WorkflowType.Conditional, @@ -859,13 +934,17 @@ describe('multirespondent-submission.utils', () => { const mockResponsesB = { [mockConditionalFieldId]: { fieldType: BasicField.Dropdown, - answer: 'Option B', + answer: { value: 'Option B' }, + question: 'Dropdown', + provenance: {}, }, [mockShortTextFieldId]: { fieldType: BasicField.ShortText, - answer: 'Some text response', + answer: { value: 'Some text response' }, + question: 'Short Text', + provenance: {}, }, - } as FieldResponsesV3 + } as any // Act & Assert for Option B const resultB = retrieveWorkflowStepEmailAddresses( @@ -1080,9 +1159,11 @@ describe('multirespondent-submission.utils', () => { responses: { [fieldId]: { fieldType: BasicField.ShortText, - answer: 'Alice', - } as ShortTextResponseV3, - }, + answer: { value: 'Alice' }, + question: 'Name', + provenance: {}, + }, + } as any, }), ) expect(result[2]).toEqual({ question: 'Name', answer: 'Alice' }) @@ -1104,17 +1185,17 @@ describe('multirespondent-submission.utils', () => { [fieldId]: { fieldType: BasicField.Address, answer: { - addressSubFields: { - blockNumber: '161', - streetName: 'BUKIT BATOK STREET 11', - buildingName: '', - levelNumber: '01', - unitNumber: '02', - postalCode: '650161', - } as AddressAttributes, + blockNumber: { value: '161' }, + streetName: { value: 'BUKIT BATOK STREET 11' }, + buildingName: { value: '' }, + levelNumber: { value: '01' }, + unitNumber: { value: '02' }, + postalCode: { value: '650161' }, }, - } as AddressResponseV3, - }, + question: 'Home Address', + provenance: {}, + }, + } as any, }), ) expect(result[2]).toEqual({ @@ -1159,8 +1240,10 @@ describe('multirespondent-submission.utils', () => { [fieldId]: { fieldType: BasicField.Email, answer: { value: 'alice@example.com', signature: 'sig' }, - } as EmailResponseV3, - }, + question: 'Email', + provenance: {}, + }, + } as any, }), ) expect(result[2]).toEqual({ @@ -1215,9 +1298,11 @@ describe('multirespondent-submission.utils', () => { responses: { '4': { fieldType: BasicField.ShortText, - answer: 'Alice', - } as ShortTextResponseV3, - }, + answer: { value: 'Alice' }, + question: 'Name', + provenance: {}, + }, + } as any, }), ) expect(result).toHaveLength(3) // Response ID + Timestamp + Name @@ -1239,9 +1324,11 @@ describe('multirespondent-submission.utils', () => { responses: { [fieldId]: { fieldType: BasicField.ShortText, - answer: 'Alice', - } as ShortTextResponseV3, - }, + answer: { value: 'Alice' }, + question: 'Name', + provenance: {}, + }, + } as any, }), ) expect(result[2]).not.toHaveProperty('fieldType') diff --git a/apps/backend/src/app/modules/submission/multirespondent-submission/multirespondent-submission.middleware.ts b/apps/backend/src/app/modules/submission/multirespondent-submission/multirespondent-submission.middleware.ts index 2a8fc508cc..06cf3dbef2 100644 --- a/apps/backend/src/app/modules/submission/multirespondent-submission/multirespondent-submission.middleware.ts +++ b/apps/backend/src/app/modules/submission/multirespondent-submission/multirespondent-submission.middleware.ts @@ -1,4 +1,4 @@ -import type { FieldResponsesV4, FormFieldMeta } from '@opengovsg/formsg-sdk' +import type { FieldResponsesV4 } from '@opengovsg/formsg-sdk' import { adaptV3ToV4, adaptV4ToV3, @@ -7,9 +7,9 @@ import { import { celebrate, Joi, Segments } from 'celebrate' import crypto from 'crypto' import { NextFunction } from 'express' -import { featureFlags } from 'formsg-shared/constants' import { BasicField, + FieldResponsesV3, FormAuthType, FormDto, FormFieldDto, @@ -17,7 +17,6 @@ import { SubmissionType, } from 'formsg-shared/types' import { StatusCodes } from 'http-status-codes' -import _ from 'lodash' import { err, errAsync, ok, okAsync, Result, ResultAsync } from 'neverthrow' import { @@ -27,9 +26,9 @@ import { } from 'src/types' import { - ParsedClearAttachmentResponseV3, - ParsedClearFormFieldResponsesV3, - ParsedClearFormFieldResponseV3, + ParsedClearAttachmentFieldResponseV4, + ParsedClearFormFieldResponsesV4, + ParsedClearFormFieldResponseV4, } from '../../../../types/api' import { MultirespondentFormLoadedDto, @@ -43,7 +42,7 @@ import { getVisibleFieldIdsV3, } from '../../../utils/logic-adaptor' import { createReqMeta } from '../../../utils/request' -import { isFieldResponseV3Equal } from '../../../utils/response-v3' +import { isFieldResponseV4Equal } from '../../../utils/response-v4' import { DatabaseError } from '../../core/core.errors' import * as FeatureFlagService from '../../feature-flags/feature-flags.service' import { assertFormAvailable } from '../../form/admin-form/admin-form.utils' @@ -51,7 +50,7 @@ import * as FormService from '../../form/form.service' import { MyInfoService } from '../../myinfo/myinfo.service' import { extractMyInfoLoginJwt } from '../../myinfo/myinfo.util' import { getOidcService } from '../../spcp/spcp.oidc.service' -import { createNdiResponsesV3FromRecord } from '../../spcp/spcp.util' +import { createNdiResponsesV4FromRecord } from '../../spcp/spcp.util' import * as VerifiedContentService from '../../verified-content/verified-content.service' import { VerifiedContentV3 } from '../../verified-content/verified-content.types' import { FormsgReqBodyExistsError } from '../encrypt-submission/encrypt-submission.errors' @@ -61,7 +60,6 @@ import { MissingSubmitterIdError, MrfWorkflowOverflowError, ProcessingError, - SubmissionEncryptionMismatchError, SubmissionEncryptionVerificationError, SubmissionNotFoundError, } from '../submission.errors' @@ -69,7 +67,7 @@ import * as SubmissionService from '../submission.service' import { generateHashedSubmitterId, getEncryptedAttachmentsMapFromAttachmentsMap, - isAttachmentResponseV3, + isAttachmentResponseV4, mapRouteError, } from '../submission.utils' @@ -83,7 +81,7 @@ import { MultirespondentSubmissionMiddlewareHandlerType, ProcessedMultirespondentSubmissionHandlerRequest, ProcessedMultirespondentSubmissionHandlerType, - StrippedAttachmentResponseV3, + StrippedAttachmentResponseV4, } from './multirespondent-submission.types' import { validateMrfFieldResponses } from './multirespondent-submission.utils' @@ -94,8 +92,10 @@ const multirespondentSubmissionBodySchema = Joi.object({ /^[a-fA-F0-9]{24}$/, Joi.object({ fieldType: Joi.string().valid(...Object.values(BasicField)), - //TODO(MRF/FRM-1592): Improve this validation, should match ParsedClearFormFieldResponseV3 answer: Joi.required(), + question: Joi.any().strip(), + provenance: Joi.object().optional(), + myInfo: Joi.object({ attr: Joi.string().required() }).optional(), }), ), responseMetadata: Joi.object({ @@ -263,8 +263,8 @@ export const createFormsgAndRetrieveForm = ( }) } -type IdTaggedParsedClearAttachmentResponseV3 = - ParsedClearAttachmentResponseV3 & { id: string } +type IdTaggedParsedClearAttachmentResponseV4 = + ParsedClearAttachmentFieldResponseV4 & { id: string } /** * Asynchronous virus scanning for storage submissions v2.1+. This is used for non-dev environments. @@ -272,10 +272,10 @@ type IdTaggedParsedClearAttachmentResponseV3 = * @returns all responses with clean attachments and their filename populated for any attachment fields. */ const asyncVirusScanning = ( - responses: IdTaggedParsedClearAttachmentResponseV3[], + responses: IdTaggedParsedClearAttachmentResponseV4[], formId: string, ): ResultAsync< - IdTaggedParsedClearAttachmentResponseV3, + IdTaggedParsedClearAttachmentResponseV4, SubmissionService.TriggerGuardDutyScanThenDownloadCleanFileChainError >[] => { return responses.map((response) => { @@ -283,13 +283,17 @@ const asyncVirusScanning = ( // for us to compare the reliability of the services // use guardduty scan results - return SubmissionService.triggerGuardDutyScanThenDownloadCleanFileChain( - response.answer, + const { id, ...attachmentResponse } = response + return SubmissionService.triggerGuardDutyScanThenDownloadCleanFileChainV4( + attachmentResponse, formId, - ).map((attachmentResponse) => ({ - ...response, - answer: attachmentResponse, - })) + ).map( + (scannedResponse) => + ({ + ...scannedResponse, + id, + }) as IdTaggedParsedClearAttachmentResponseV4, + ) }) } @@ -299,27 +303,33 @@ const asyncVirusScanning = ( * @returns all responses with clean attachments and their filename populated for any attachment fields. */ const devModeSyncVirusScanning = async ( - responses: IdTaggedParsedClearAttachmentResponseV3[], + responses: IdTaggedParsedClearAttachmentResponseV4[], formId: string, ): Promise< Result< - IdTaggedParsedClearAttachmentResponseV3, + IdTaggedParsedClearAttachmentResponseV4, SubmissionService.TriggerGuardDutyScanThenDownloadCleanFileChainError >[] > => { const results = [] for (const response of responses) { // await to pause for...of loop until the virus scanning and downloading of clean file is completed. - const attachmentResponse = - await SubmissionService.triggerGuardDutyScanThenDownloadCleanFileChain( - response.answer, + const { id, ...attachmentResponse } = response + const scannedResult = + await SubmissionService.triggerGuardDutyScanThenDownloadCleanFileChainV4( + attachmentResponse, formId, ) - if (attachmentResponse.isErr()) { - results.push(err(attachmentResponse.error)) + if (scannedResult.isErr()) { + results.push(err(scannedResult.error)) break } - results.push(ok({ ...response, answer: attachmentResponse.value })) + results.push( + ok({ + ...scannedResult.value, + id, + } as IdTaggedParsedClearAttachmentResponseV4), + ) } return results } @@ -338,7 +348,7 @@ export const scanAndRetrieveAttachments = async ( } // Step 1: Extract attachment responses into an array to prepare for virus scanning. - const attachmentResponsesToRetrieve: IdTaggedParsedClearAttachmentResponseV3[] = + const attachmentResponsesToRetrieve: IdTaggedParsedClearAttachmentResponseV4[] = Object.keys(req.body.responses) .map((id) => { const response = req.body.responses[id] @@ -349,10 +359,13 @@ export const scanAndRetrieveAttachments = async ( ) { return null } - return { id, ...response } + return { + id, + ...response, + } as unknown as IdTaggedParsedClearAttachmentResponseV4 }) .filter( - (value): value is IdTaggedParsedClearAttachmentResponseV3 => + (value): value is IdTaggedParsedClearAttachmentResponseV4 => value !== null, ) @@ -399,7 +412,9 @@ export const scanAndRetrieveAttachments = async ( // Step 3: Update responses with new values. for (const idTaggedAttachmentResponse of scanAndRetrieveFilesResult.value) { - const { id, ...attachmentResponse } = idTaggedAttachmentResponse + const { id, ...attachmentResponseRaw } = idTaggedAttachmentResponse + const attachmentResponse = + attachmentResponseRaw as ParsedClearAttachmentFieldResponseV4 // TODO: FRM-1839 Skip scanning if attachment has already been scanned attachmentResponse.answer.hasBeenScanned = true // Store the md5 hash in the DB as well for comparison later on. @@ -408,7 +423,8 @@ export const scanAndRetrieveAttachments = async ( .update(Buffer.from(attachmentResponse.answer.content)) .digest() .toString() - req.body.responses[id] = attachmentResponse + req.body.responses[id] = + attachmentResponse as unknown as FieldResponsesV4[string] } return next() @@ -505,14 +521,19 @@ export const validateMultirespondentSubmission = async ( form_logics, } as Pick + // Convert V4 responses to V3 for logic evaluation (logic engine uses V3 format) + const responsesV3ForLogic = adaptV4ToV3( + req.body.responses as FieldResponsesV4, + ) as unknown as FieldResponsesV3 + // Step 0c: Get visible fields based on evaluation of logic return getVisibleFieldIdsV3( - req.body.responses, + responsesV3ForLogic, formPropertiesForLogicComputation, ).andThen((visibleFieldIds) => // Step 1: Check prevent submission logic getLogicUnitPreventingSubmitV3( - req.body.responses, + responsesV3ForLogic, formPropertiesForLogicComputation, visibleFieldIds, ) @@ -573,19 +594,26 @@ export const validateMultirespondentSubmission = async ( } /** - * Since the incoming client responses are in V3, - * if previous submission was encrypted in V4 format, convert to V3 - * to facilitate comparison. + * Since the incoming client responses are in V4, + * if previous submission was encrypted in V3 format, convert to V4 + * to facilitate comparison. Comparison (isFieldResponseV4Equal) + * only inspects fieldType + answer, so we skip the formFields + * meta — question text and myInfo on the adapted response are + * unused; downstream consumers source both from the form field. */ const previousResponses = (() => { const responses = previousSubmissionDecryptedContent.responses - if (isFieldResponsesV4(responses)) { - return adaptV4ToV3( - responses as FieldResponsesV4, - ) as ParsedClearFormFieldResponsesV3 + if ( + !isFieldResponsesV4(responses as Record) + ) { + // Response is in V3 format, adapt to V4 + return adaptV3ToV4(responses, { + formFields: {}, + provenance: {}, + }) as ParsedClearFormFieldResponsesV4 } - // Response is in v3 format, return as is - return responses as ParsedClearFormFieldResponsesV3 + // Response is in V4 format, return as is + return responses as unknown as ParsedClearFormFieldResponsesV4 })() const previousNonEditableFieldIdsWithResponses = Object.keys( @@ -604,20 +632,9 @@ export const validateMultirespondentSubmission = async ( const incomingResField = req.body.responses[fieldId] const prevResField = previousResponses[fieldId] - if ( - prevResField.fieldType === BasicField.ShortText || - prevResField.fieldType === BasicField.LongText - ) { - // NOTE: LEGACY ISSUE - // Since text fields were saved without trimming prior to https://github.com/opengovsg/FormSG/pull/7937. - // Without this, isFieldResponseV3Equal fails since the prevResField was not trimmed, - // causing a mismatch between the newly trimmed incomingResField. - prevResField.answer = prevResField.answer.trim() - } - - const resp = isFieldResponseV3Equal( - incomingResField, - prevResField, + const resp = isFieldResponseV4Equal( + incomingResField as FieldResponsesV4[string], + prevResField as FieldResponsesV4[string], ) if (!resp) { @@ -646,11 +663,9 @@ export const validateMultirespondentSubmission = async ( incomingResField.fieldType === BasicField.Attachment && prevResField.fieldType === BasicField.Attachment ) { - incomingResField.answer.answer = - prevResField.answer.answer - - incomingResField.answer.filename = - prevResField.answer.answer + ;(incomingResField.answer as { value: string }).value = ( + prevResField.answer as { value: string } + ).value } return ok(undefined) @@ -759,14 +774,12 @@ export const encryptSubmission = async ( res: Parameters[1], next: NextFunction, ) => { - const formDef = req.formsg.formDef - void req.growthbook?.setAttributes({ ...req.growthbook.getAttributes(), formId: req.params.formId, - formCreated: formDef.created?.toISOString(), }) + const formDef = req.formsg.formDef const formPublicKey = formDef.publicKey const responses = req.body.responses @@ -774,7 +787,7 @@ export const encryptSubmission = async ( const strippedAttachmentResponses: Record< string, - ParsedClearFormFieldResponseV3 | StrippedAttachmentResponseV3 + ParsedClearFormFieldResponseV4 | StrippedAttachmentResponseV4 > = {} const unencryptedAttachments: IAttachmentInfo[] = [] @@ -782,65 +795,45 @@ export const encryptSubmission = async ( // Populate attachment map for (const id of Object.keys(responses)) { const response = responses[id] - if (response.fieldType !== BasicField.Attachment) { + if (!isAttachmentResponseV4(response)) { strippedAttachmentResponses[id] = response continue } attachmentsMap[id] = response.answer.content - strippedAttachmentResponses[id] = { - ...response, - answer: { ...response.answer, filename: undefined, content: undefined }, - } + const attachmentRes = response as ParsedClearAttachmentFieldResponseV4 + const strippedAttachment = { + ...attachmentRes, + answer: { + value: attachmentRes.answer.value, + hasBeenScanned: attachmentRes.answer.hasBeenScanned, + md5Hash: attachmentRes.answer.md5Hash, + filename: undefined, + content: undefined, + }, + } as unknown as StrippedAttachmentResponseV4 + strippedAttachmentResponses[id] = strippedAttachment // collect unencrypted attachments to include in email notifications - if (isAttachmentResponseV3(response)) { - unencryptedAttachments.push({ - filename: response.answer.filename, - content: response.answer.content, - fieldId: id, - }) - } + unencryptedAttachments.push({ + filename: response.answer.filename, + content: response.answer.content, + fieldId: id, + }) } if (req.formsg) { req.formsg.unencryptedAttachments = unencryptedAttachments } - const useV4Encryption = - (req.growthbook?.isOn(featureFlags.answerObjectEncryption) && - !formDef.webhook?.url) ?? - false - - let responsesToEncrypt: - | Record< - string, - ParsedClearFormFieldResponseV3 | StrippedAttachmentResponseV3 - > - | FieldResponsesV4 = strippedAttachmentResponses - - if (useV4Encryption) { - logger.info({ - message: 'Using V4 encryption for submission', - meta: { - action: 'encryptSubmission', - formId: req.params.formId, - submissionId: req.params.submissionId, - }, - }) - // Build FormFieldMeta map from form definition for question text - const formFieldMeta: Record = {} - for (const field of formDef.form_fields) { - formFieldMeta[field._id] = { - question: field.title, - ...(field.myInfo?.attr && { myInfo: { attr: field.myInfo.attr } }), - } - } - - responsesToEncrypt = adaptV3ToV4(strippedAttachmentResponses, { - formFields: formFieldMeta, - provenance: {}, - }) - } + // Webhook compatibility: forms with webhooks have downstream consumers that + // parse the encrypted payload as V3-shaped (mrfVersion: 1). Convert back to + // V3 just for the encryption blob; in-process state (encryptedPayload.responses, + // emails, NDI handling) stays V4. + const hasWebhook = !!formDef.webhook?.url + const responsesToEncrypt = hasWebhook + ? adaptV4ToV3(strippedAttachmentResponses as unknown as FieldResponsesV4) + : strippedAttachmentResponses + const mrfVersion: 1 | 2 = hasWebhook ? 1 : 2 const { encryptedContent, @@ -850,7 +843,6 @@ export const encryptSubmission = async ( } = formsgSdk.cryptoV3.encrypt(responsesToEncrypt, formPublicKey) // Verify the encrypted content can be decrypted using the generated submission secret key before saving - // Perform a diff check between original & recently decrypted responses to ensure round trip encryption-decryption const decryptionVerification = formsgSdk.cryptoV3.decryptFromSubmissionKey( submissionSecretKey, { encryptedContent, version: req.body.version }, @@ -864,7 +856,7 @@ export const encryptSubmission = async ( formId: req.params.formId, submissionId: req.params.submissionId, version: req.body.version, - mrfVersion: useV4Encryption ? 2 : 1, + mrfVersion, ...createReqMeta(req), }, error, @@ -874,41 +866,6 @@ export const encryptSubmission = async ( }) } - const decryptedResponsesV3 = isFieldResponsesV4( - decryptionVerification.responses, - ) - ? adaptV4ToV3(decryptionVerification.responses as FieldResponsesV4) - : decryptionVerification.responses - - // cryptoV3.encrypt serializes via JSON.stringify, which drops `undefined` object keys. - const normalizedOriginalResponses = JSON.parse( - JSON.stringify(strippedAttachmentResponses), - ) as typeof strippedAttachmentResponses - - const responseMismatch = !_.isEqual( - normalizedOriginalResponses, - decryptedResponsesV3, - ) - - if (responseMismatch) { - const mismatchError = new SubmissionEncryptionMismatchError() - logger.error({ - message: mismatchError.message, - meta: { - action: 'encryptSubmission', - formId: req.params.formId, - submissionId: req.params.submissionId, - version: req.body.version, - mrfVersion: useV4Encryption ? 2 : 1, - ...createReqMeta(req), - }, - error: mismatchError, - }) - return res.status(StatusCodes.INTERNAL_SERVER_ERROR).json({ - message: 'An error occurred while processing your submission', - }) - } - const encryptedAttachments = await getEncryptedAttachmentsMapFromAttachmentsMap( attachmentsMap, @@ -925,12 +882,13 @@ export const encryptSubmission = async ( submissionSecretKey, version: req.body.version, workflowStep: req.body.workflowStep, - responses, + responses: responses as FieldResponsesV4, /** - * MRF Version: 1 — V3 encrypted responses - * MRF Version: 2 — V4 encrypted responses (with provenance) + * MRF Version 2 = V4-encrypted responses (with provenance). + * MRF Version 1 = V3-encrypted responses (used when form has a webhook + * so existing webhook consumers can continue to parse V3 payloads). */ - mrfVersion: useV4Encryption ? 2 : 1, + mrfVersion, } return next() @@ -1091,7 +1049,7 @@ export const handleNdiResponses = async ( } // 3. Add collected Ndi data to responses for email payload - const emailNdiResponses = createNdiResponsesV3FromRecord(ndiResponses) + const emailNdiResponses = createNdiResponsesV4FromRecord(ndiResponses) responses = { ...responses, ...emailNdiResponses } req.formsg.encryptedPayload.responses = responses diff --git a/apps/backend/src/app/modules/submission/multirespondent-submission/multirespondent-submission.service.ts b/apps/backend/src/app/modules/submission/multirespondent-submission/multirespondent-submission.service.ts index d8c2285639..7cac716ebc 100644 --- a/apps/backend/src/app/modules/submission/multirespondent-submission/multirespondent-submission.service.ts +++ b/apps/backend/src/app/modules/submission/multirespondent-submission/multirespondent-submission.service.ts @@ -1,7 +1,7 @@ import { GrowthBook } from '@growthbook/growthbook' +import type { FieldResponsesV4 } from '@opengovsg/formsg-sdk' import { BasicField, - FieldResponsesV3, FormAuthType, FormFieldDto, FormResponseMode, @@ -122,7 +122,7 @@ const checkIsStepRejected = ({ }: { zeroIndexedStepNumber: number form: Pick - responses: FieldResponsesV3 + responses: FieldResponsesV4 }): Result< boolean, ExpectedResponseNotFoundError | InvalidApprovalFieldTypeError @@ -146,7 +146,7 @@ const checkIsStepRejected = ({ return err(new InvalidApprovalFieldTypeError()) } - return ok(approvalFieldResponse.answer === 'No') + return ok((approvalFieldResponse.answer as { value: string }).value === 'No') } interface sendNextStepEmailProps { @@ -158,7 +158,7 @@ interface sendNextStepEmailProps { responseUrl: string formId: string submissionId: string - responses: FieldResponsesV3 + responses: FieldResponsesV4 } const sendNextStepEmail = ({ @@ -353,7 +353,7 @@ const getEmailsToNotifyAboutMrfOutcome = ({ > & { form_fields: FormFieldSchema[] | FormFieldDto[] } - responses: FieldResponsesV3 + responses: FieldResponsesV4 currentStepNumber: number submissionId: string }): Result => { @@ -456,7 +456,7 @@ const sendMrfOutcomeEmails = ({ > & { form_fields: FormFieldSchema[] | FormFieldDto[] } - responses: FieldResponsesV3 + responses: FieldResponsesV4 latestSubmissionTimestamp: string submissionId: string isApproval?: boolean @@ -605,7 +605,7 @@ const sendMrfRespondentCopyEmails = ({ > & { form_fields: FormFieldSchema[] | FormFieldDto[] } - responses: FieldResponsesV3 + responses: FieldResponsesV4 submission: IMultirespondentSubmissionSchema attachments?: IAttachmentInfo[] formFields: FormFieldSchema[] | FormFieldDto[] @@ -860,7 +860,7 @@ export const createMultiRespondentFormSubmission = ({ } interface CheckIfRespondentFormSummaryIsRequiredArgs { - responses: FieldResponsesV3 + responses: FieldResponsesV4 formFields: FormFieldSchema[] | FormFieldDto[] currentStepActiveFields: string[] } @@ -893,7 +893,7 @@ interface CheckIsWorkflowCompletionEmailPdfRequiredArgs { > & { form_fields: FormFieldSchema[] | FormFieldDto[] } - responses: FieldResponsesV3 + responses: FieldResponsesV4 isRejected: boolean submissionId: string growthbook?: GrowthBook diff --git a/apps/backend/src/app/modules/submission/multirespondent-submission/multirespondent-submission.types.ts b/apps/backend/src/app/modules/submission/multirespondent-submission/multirespondent-submission.types.ts index 4835fa87de..31a553b33a 100644 --- a/apps/backend/src/app/modules/submission/multirespondent-submission/multirespondent-submission.types.ts +++ b/apps/backend/src/app/modules/submission/multirespondent-submission/multirespondent-submission.types.ts @@ -1,6 +1,5 @@ import type { AttachmentAnswerV4 } from '@opengovsg/formsg-sdk' import { - AttachmentResponseV3, MyInfoAttribute, SubmissionErrorDto, SubmissionResponseDto, @@ -13,7 +12,7 @@ import { MultirespondentFormCompleteDto, MultirespondentFormLoadedDto, ParsedClearAttachmentFieldResponseV4, - ParsedClearFormFieldResponsesV3, + ParsedClearFormFieldResponsesV4, ParsedMultirespondentSubmissionBody, } from '../../../../types/api' import { ControllerHandler } from '../../core/core.types' @@ -48,7 +47,7 @@ export type ProcessedMultirespondentSubmissionHandlerType = ControllerHandler< SubmissionResponseDto | SubmissionErrorDto, Omit & { submissionSecretKey?: string - responses: ParsedClearFormFieldResponsesV3 + responses: ParsedClearFormFieldResponsesV4 }, { captchaResponse?: unknown; captchaType?: unknown } > @@ -106,13 +105,6 @@ export type StrippedAttachmentResponseV4 = Omit< } } -export type StrippedAttachmentResponseV3 = AttachmentResponseV3 & { - answer: AttachmentResponseV3['answer'] & { - filename: undefined - content: undefined - } -} - export type MrfJwtPayload = { prevSubmissionId: string currentWorkflowStep: number diff --git a/apps/backend/src/app/modules/submission/multirespondent-submission/multirespondent-submission.utils.ts b/apps/backend/src/app/modules/submission/multirespondent-submission/multirespondent-submission.utils.ts index 9919c7d38e..c3a43f5785 100644 --- a/apps/backend/src/app/modules/submission/multirespondent-submission/multirespondent-submission.utils.ts +++ b/apps/backend/src/app/modules/submission/multirespondent-submission/multirespondent-submission.utils.ts @@ -1,22 +1,16 @@ +import type { FieldResponsesV4, FieldResponseV4 } from '@opengovsg/formsg-sdk' import { CLIENT_CHECKBOX_OTHERS_INPUT_VALUE } from 'formsg-shared/constants' import { BasicField, - FieldResponsesV3, - FieldResponseV3, FormFieldDto, FormMetadata, FormWorkflowStepDto, MultirespondentSubmissionDto, - NdiResponseV3, PublicMultirespondentSubmissionDto, SubmissionType, WorkflowType, } from 'formsg-shared/types' -import { - answerKey, - handleAddressResponseDisplay, -} from 'formsg-shared/utils/address' -import { NON_RESPONSE_FIELD_SET } from 'formsg-shared/utils/field' +import { handleAddressResponseDisplay } from 'formsg-shared/utils/address' import { SIGNATURE_CAPTURED_STRING } from 'formsg-shared/utils/signature' import { stripDropdownFieldOptionsToRecipientsMap } from 'formsg-shared/utils/strip-dropdown-field-optionsToRecipientsMap' import { stripWorkflowEmails } from 'formsg-shared/utils/strip-workflow-emails' @@ -30,18 +24,18 @@ import { IMultirespondentSubmissionSchema, MultirespondentSubmissionData, } from '../../../../types' -import { ParsedClearFormFieldResponsesV3 } from '../../../../types/api' +import { ParsedClearFormFieldResponsesV4 } from '../../../../types/api' import config from '../../../config/config' import { spcpMyInfoConfig } from '../../../config/features/spcp-myinfo.config' import { AutoReplyMailData } from '../../../services/mail/mail.types' import { convertToSignaturePngDataUri } from '../../../utils/convert-vector-array-to-png' -import { validateFieldV3 } from '../../../utils/field-validation' +import { validateFieldV4 } from '../../../utils/field-validation' import { FieldIdSet } from '../../../utils/logic-adaptor' import { startsWithSPCPFieldTitle } from '../../spcp/spcp.util' import { InvalidWorkflowTypeError, ProcessingError, - ValidateFieldErrorV3, + ValidateFieldErrorV4, } from '../submission.errors' import { buildMrfMetadata } from '../submission.utils' @@ -105,39 +99,29 @@ export const createPublicMultirespondentSubmissionDto = ( } } -export const getFormDelimiter = (metadata?: FormMetadata): string => - metadata?.delimiter ?? ', ' - export const getEmailFromResponses = ( fieldId: string, - responses: FieldResponsesV3, + responses: FieldResponsesV4, ): string | null => { const field = responses[fieldId] if (!field || field.fieldType !== BasicField.Email) return null // Not an error, misconfigured or respondent has not filled. - return field.answer.value + return (field.answer as { value: string }).value } export const extractEmailAnswersFromResponses = ( - responses: FieldResponsesV3, + responses: FieldResponsesV4, ): string[] => { if (!responses) return [] return Object.values(responses) - .filter( - ( - response, - ): response is Extract< - FieldResponseV3, - { fieldType: BasicField.Email } - > => response.fieldType === BasicField.Email, - ) - .map((response) => response.answer.value) + .filter((response) => response.fieldType === BasicField.Email) + .map((response) => (response.answer as { value: string }).value) .filter(Boolean) } const getConditionalFieldEmailRecipient = ( form_fields: FormFieldSchema[] | FormFieldDto[], fieldId: string, - responses: FieldResponsesV3, + responses: FieldResponsesV4, ): string[] => { const conditionalField = form_fields.find( (field) => field._id.toString() === fieldId.toString(), @@ -157,10 +141,10 @@ const getConditionalFieldEmailRecipient = ( return [] // Not an error, misconfigured or respondent has not filled. } + const answerValue = (conditionalFieldResponse.answer as { value: string }) + .value const emailRecipients = - conditionalField?.optionsToRecipientsMap?.[ - conditionalFieldResponse.answer - ] ?? [] + conditionalField?.optionsToRecipientsMap?.[answerValue] ?? [] return emailRecipients } @@ -168,7 +152,7 @@ const getConditionalFieldEmailRecipient = ( export const retrieveWorkflowStepEmailAddresses = ( form: { form_fields: FormFieldSchema[] | FormFieldDto[] }, step: FormWorkflowStepDto, - responses: FieldResponsesV3, + responses: FieldResponsesV4, ): Result => { if (!step) return ok([]) // Not an error, just that the form has gone past its predefined workflow switch (step.workflow_type) { @@ -212,11 +196,11 @@ export const validateMrfFieldResponses = ({ formId: string visibleFieldIds: FieldIdSet formFields: FormFieldDto[] - responses: ParsedClearFormFieldResponsesV3 - previousResponses?: ParsedClearFormFieldResponsesV3 + responses: ParsedClearFormFieldResponsesV4 + previousResponses?: ParsedClearFormFieldResponsesV4 }): Result< - ParsedClearFormFieldResponsesV3, - ValidateFieldErrorV3 | ProcessingError + ParsedClearFormFieldResponsesV4, + ValidateFieldErrorV4 | ProcessingError > => { const idToFieldMap = formFields.reduce<{ [fieldId: string]: FormFieldDto @@ -236,21 +220,21 @@ export const validateMrfFieldResponses = ({ // Since Myinfo fields are not currently supported for MRF if (response.fieldType === BasicField.Children) { return err( - new ValidateFieldErrorV3( + new ValidateFieldErrorV4( 'Children field type is not supported for MRF submisisons', ), ) } - const validateFieldV3Result = validateFieldV3({ + const validateFieldV4Result = validateFieldV4({ formId, formField, response, prevResponse: previousResponses?.[responseId], isVisible: visibleFieldIds.has(responseId), }) - if (validateFieldV3Result.isErr()) { - return err(validateFieldV3Result.error) + if (validateFieldV4Result.isErr()) { + return err(validateFieldV4Result.error) } } @@ -269,7 +253,7 @@ export const extractRespondentCopyEmailDatas = ({ formFields, currentStepActiveFields, }: { - responses: FieldResponsesV3 + responses: FieldResponsesV4 formFields: FormFieldSchema[] | FormFieldDto[] currentStepActiveFields: string[] }): AutoReplyMailData[] => { @@ -284,10 +268,10 @@ export const extractRespondentCopyEmailDatas = ({ field.fieldType === BasicField.Email && field.autoReplyOptions?.hasAutoReply && response && - // checks if response has an answer (email) + // checks if response has an answer (email) - V4 email answer is always { value: string } typeof response.answer === 'object' && 'value' in response.answer && - typeof response.answer.value === 'string' + typeof (response.answer as { value: string }).value === 'string' ) { const { autoReplyMessage, @@ -297,7 +281,7 @@ export const extractRespondentCopyEmailDatas = ({ } = field.autoReplyOptions return [ { - email: response.answer.value, + email: (response.answer as { value: string }).value, subject: autoReplySubject, sender: autoReplySender, body: autoReplyMessage, @@ -317,77 +301,84 @@ export type QuestionAnswerPair = { } /** - * Given a single form field and its response, extracts question-answer pairs. + * Given a single form field and its response (V4), extracts question-answer pairs. * Used for email body/pdf outputs and individualResponsePage displays * Returns an array since some fields (e.g. table, children) will have * multiple question-answer pairs per response * @param formField - Single form field schema. Does not include Ndi responses, @see getQuestionAnswerPairsForMultipleFields on how to include ndi responses. - * @param response - Response for the given form field + * @param response - V4 Response for the given form field * @returns An array of QuestionAnswer objects representing the extracted question-answer pairs for the given form field. */ const getQuestionAnswerPairsForOneField = ({ formField, response, includeSignatureDataPngDataUri, - delimiter = '; ', + includeVerifiedPrefix = true, }: { formField: FormFieldSchema | FormFieldDto - response: FieldResponseV3 + response: FieldResponseV4 includeSignatureDataPngDataUri: boolean - delimiter?: string + includeVerifiedPrefix?: boolean }): QuestionAnswerPair[] => { - let questionTitle = formField.title + // V4 responses embed question text in response.question + let questionTitle = response.question || formField.title let answer = '' - let answerArray: string[] = [] const questionAnswerPairs: QuestionAnswerPair[] = [] - // edge case handling for headers - if (formField.fieldType === BasicField.Section) { - questionAnswerPairs.push({ - question: questionTitle, - answer, - fieldType: formField.fieldType, - }) - return questionAnswerPairs - } - + const fieldType = response.fieldType as BasicField switch (response.fieldType) { - case BasicField.Attachment: - questionTitle = `[Attachment] ${questionTitle}` - answer = response.answer.answer - break + case BasicField.Attachment: { + const attachmentAnswer = response.answer as { + value: string + hasBeenScanned: boolean + md5Hash?: string + } + return [ + { + question: `[Attachment] ${questionTitle}`, + answer: attachmentAnswer.value, + fieldType, + }, + ] + } case BasicField.Address: { - const { - postalCode, - blockNumber, - streetName, - buildingName, - levelNumber, - unitNumber, - } = response.answer.addressSubFields - answerArray = [ - blockNumber, - streetName, - buildingName, - levelNumber, - unitNumber, - postalCode, + const addressAnswer = response.answer as { + postalCode: { value: string } + blockNumber: { value: string } + streetName: { value: string } + buildingName: { value: string } + levelNumber: { value: string } + unitNumber: { value: string } + } + const answerArray = [ + addressAnswer.blockNumber.value, + addressAnswer.streetName.value, + addressAnswer.buildingName.value, + addressAnswer.levelNumber.value, + addressAnswer.unitNumber.value, + addressAnswer.postalCode.value, ] // move postal code to end of array - answer = handleAddressResponseDisplay(Object.values(answerArray)).join( - ', ', - ) + answer = handleAddressResponseDisplay(answerArray).join(', ') break } case BasicField.Email: - case BasicField.Mobile: - if (response.answer.signature) + case BasicField.Mobile: { + const verifiableAnswer = response.answer as { + value: string + signature?: string + } + if (includeVerifiedPrefix && verifiableAnswer.signature) { questionTitle = `[Verified] ${questionTitle}` - - answer = response.answer.value + } + answer = verifiableAnswer.value break - case BasicField.Table: - if (formField.fieldType !== BasicField.Table || !response.answer) break - // eslint-disable-next-line no-case-declarations + } + case BasicField.Table: { + if (formField.fieldType !== BasicField.Table) break + const tableAnswer = response.answer as Record< + string, + { rowNum: number; value: Record } + > const idToColTitleMap = formField.columns.reduce( (acc, col) => { acc[col._id] = col.title @@ -396,8 +387,13 @@ const getQuestionAnswerPairsForOneField = ({ {} as Record, ) - for (const row of response.answer) { - const validColumns = Object.entries(row).filter( + // Sort rows by rowNum for consistent ordering + const sortedRows = Object.values(tableAnswer).sort( + (a, b) => a.rowNum - b.rowNum, + ) + + for (const row of sortedRows) { + const validColumns = Object.entries(row.value).filter( ([colId]) => colId in idToColTitleMap, ) @@ -406,76 +402,76 @@ const getQuestionAnswerPairsForOneField = ({ const colTitle = idToColTitleMap[colId] return `${colTitle}` }) - .join(delimiter) + .join('; ') const delimitedColumnAnswers = validColumns - .map(([, colAns]) => colAns ?? '') - .join(delimiter) + .map(([, colAns]) => + colAns !== null && colAns !== undefined ? String(colAns) : '', + ) + .join('; ') const question = `[Table] ${formField.title} (${delimitedColumnTitles})` - const answer = delimitedColumnAnswers questionAnswerPairs.push({ question, - answer, - fieldType: response.fieldType, + answer: delimitedColumnAnswers, + fieldType, }) } return questionAnswerPairs - case BasicField.Radio: - answer = - 'value' in response.answer - ? response.answer.value - : 'othersInput' in response.answer - ? 'Others: ' + response.answer.othersInput - : '' + } + case BasicField.Radio: { + const radioAnswer = response.answer as { + value: string + isOthersInput: boolean + } + if (radioAnswer.isOthersInput) { + answer = `Others: ${radioAnswer.value}` + } else { + answer = radioAnswer.value + } break + } case BasicField.Checkbox: { - answer = response.answer.value - .map((val) => - val === CLIENT_CHECKBOX_OTHERS_INPUT_VALUE - ? response.answer.othersInput - ? 'Others: ' + response.answer.othersInput - : null - : val, - ) - .filter(Boolean) - .join(', ') + const checkboxAnswer = response.answer as { + value: string[] + othersInput?: string + } + const selectedAnswers = checkboxAnswer.value.filter( + (val) => val !== CLIENT_CHECKBOX_OTHERS_INPUT_VALUE, + ) + + if (checkboxAnswer.othersInput) { + selectedAnswers.push(checkboxAnswer.othersInput) + } + + answer = selectedAnswers.join(', ') break } case BasicField.Signature: { - const signatureQuestionAnswer = { - question: `[Signature] ${questionTitle}`, + const signatureAnswer = response.answer as { + value: [number, number, number][][] + type: string + } + const signatureQuestionAnswer: QuestionAnswerPair = { + question: `[signature] ${questionTitle}`, answer: SIGNATURE_CAPTURED_STRING, signatureDataPngDataUri: includeSignatureDataPngDataUri - ? convertToSignaturePngDataUri(response.answer.value) + ? convertToSignaturePngDataUri(signatureAnswer.value) : undefined, - fieldType: response.fieldType, + fieldType, } return [signatureQuestionAnswer] } - case BasicField.Children: - if (!response.answer.childFields || !response.answer.child) { - break - } - for (const [index, child] of response.answer.child.entries()) { - questionAnswerPairs.push({ - question: `Child ${index + 1}: ${response.answer.childFields.toString()}`, - answer: child - ? child.toString() - : response.answer.childFields.map(() => '').toString(), - fieldType: response.fieldType, - }) - } - return questionAnswerPairs default: - answer = response.answer + // For all string-answer fields (number, decimal, text, homeNo, dropdown, rating, nric, uen, date, countryRegion, section, yesNo) + answer = (response.answer as { value: string }).value ?? '' } questionAnswerPairs.push({ question: questionTitle, answer, - fieldType: response.fieldType, + fieldType, }) return questionAnswerPairs } @@ -490,10 +486,12 @@ export const getQuestionAnswerPairsForMultipleFields = ({ formFields, responses, includeSignatureDataPngDataUri = false, + includeVerifiedPrefix = true, }: { formFields: FormFieldSchema[] | FormFieldDto[] - responses: FieldResponsesV3 + responses: FieldResponsesV4 includeSignatureDataPngDataUri?: boolean + includeVerifiedPrefix?: boolean }): QuestionAnswerPair[] => { const questionAnswerPairs: QuestionAnswerPair[] = [] if (!formFields || !responses) { @@ -503,143 +501,158 @@ export const getQuestionAnswerPairsForMultipleFields = ({ const questionTitle = currentFormField.title const response = responses[currentFormField._id] - if ( - (!response && currentFormField.fieldType !== BasicField.Section) || //Allow headers to be included - !questionTitle - ) - continue - + if (!response || !questionTitle) continue const questionAnswerPairsForCurrentFormField = getQuestionAnswerPairsForOneField({ formField: currentFormField, response, includeSignatureDataPngDataUri, + includeVerifiedPrefix, }) questionAnswerPairs.push(...questionAnswerPairsForCurrentFormField) } - // Add Ndi responses if they exist + // Add Ndi responses if they exist (keyed by SPCP field title in both V3 and V4) for (const key in responses) { if (startsWithSPCPFieldTitle(key)) { - const { answer, fieldType } = responses[key] as NdiResponseV3 + const ndiResponse = responses[key] + const answerValue = (ndiResponse.answer as { value: string }).value questionAnswerPairs.push({ question: key, - answer, - fieldType, + answer: answerValue, + fieldType: ndiResponse.fieldType as unknown as BasicField, }) } } return questionAnswerPairs } -/** - * Prepares responses data from MRF responses to PDF html format - * @param formFields - The form fields schema - * @param responses - The mrf responses to the form fields - * @returns list of EmailRespondentConfirmationField used for email & pdf generation - */ -export const getResponsesDataFromMrfResponses = ({ - formFields, - responses, -}: { - formFields: FormFieldSchema[] | FormFieldDto[] - responses: FieldResponsesV3 -}): EmailRespondentConfirmationField[] => { - if (!formFields || !responses) return [] - - const questionAnswerPairs = getQuestionAnswerPairsForMultipleFields({ - formFields, - responses, - includeSignatureDataPngDataUri: true, - }) +export const getFormDelimiter = (metadata?: FormMetadata): string => + metadata?.delimiter ?? ', ' - return questionAnswerPairs.map((questionAnswerPair) => { - return { - question: questionAnswerPair.question, - answerTemplate: [questionAnswerPair.answer], - answer: questionAnswerPair.signatureDataPngDataUri, - fieldType: questionAnswerPair.fieldType, - } - }) -} +// Field types that don't carry respondent answers; excluded from JSON dump. +const NON_RESPONSE_FIELD_TYPES = new Set([ + BasicField.Section, + BasicField.Statement, + BasicField.Image, +]) /** - * Serialises MRF submission responses into a JSON string for email attachment. - * Built directly from raw form fields and responses so that specific field types - * (e.g. Address) can be handled in custom ways than the generic question-answer pair extraction - * fields are handled consistently with the email body. + * Builds the JSON response payload attached to MRF completion emails. + * + * Unlike getQuestionAnswerPairsForMultipleFields (which targets human-facing + * email/PDF output), this dump targets machine consumers and so: + * - emits an empty-string entry for every form field without a response + * (schema-complete output), and + * - flattens address answers to one entry per sub-field + * (e.g. "Home Address - blockNumber") so each sub-field is individually + * addressable downstream. + * + * Note: the `delimiter` param is accepted for caller compatibility but is + * currently unused post-V4 migration. See follow-up to restore admin + * `metadata.delimiter` customisation if needed. */ export const buildMrfResponseJson = ({ formFields, responses, responseId, timestamp, - delimiter = ', ', }: { formFields: FormFieldSchema[] | FormFieldDto[] - responses: FieldResponsesV3 + responses: FieldResponsesV4 responseId: string timestamp: string delimiter?: string }): string => { - const entries: Array<{ question: string; answer: string }> = [ + const entries: { question: string; answer: string }[] = [ { question: 'Response ID', answer: responseId }, { question: 'Timestamp', answer: timestamp }, ] - if (!formFields || !responses) return JSON.stringify(entries) + if (!formFields || !responses) { + return JSON.stringify(entries) + } for (const field of formFields) { - // Skip non-response fields and fields without titles - if (NON_RESPONSE_FIELD_SET.has(field.fieldType as BasicField)) continue - if (!field.title) continue + if (NON_RESPONSE_FIELD_TYPES.has(field.fieldType)) continue + + const response = responses[field._id.toString()] - const response = responses[field._id] if (!response) { entries.push({ question: field.title, answer: '' }) continue } - switch (response.fieldType) { - case BasicField.Address: { - const subFields = response.answer.addressSubFields - for (const key of answerKey) { - entries.push({ - question: `${field.title} - ${key}`, - answer: subFields[key as keyof typeof subFields], - }) - } - break - } - case BasicField.Email: - case BasicField.Mobile: - entries.push({ question: field.title, answer: response.answer.value }) - break - default: { - const pairs = getQuestionAnswerPairsForOneField({ - formField: field, - response, - includeSignatureDataPngDataUri: false, - delimiter, + if (response.fieldType === BasicField.Address) { + const addressAnswer = response.answer as Record + for (const subField of Object.keys(addressAnswer)) { + entries.push({ + question: `${field.title} - ${subField}`, + answer: addressAnswer[subField]?.value ?? '', }) - for (const pair of pairs) { - entries.push({ question: pair.question, answer: pair.answer }) - } } + continue } - } - for (const key in responses) { - if (startsWithSPCPFieldTitle(key)) { - const { answer } = responses[key] as NdiResponseV3 - entries.push({ question: key, answer }) + const pairs = getQuestionAnswerPairsForMultipleFields({ + formFields: [field] as unknown as FormFieldDto[], + responses: { + [field._id.toString()]: response, + } as unknown as FieldResponsesV4, + includeSignatureDataPngDataUri: false, + // [Verified] prefix is a UX hint for the email body; the JSON dump is + // for machine consumers and should carry the raw question text. + includeVerifiedPrefix: false, + }) + for (const p of pairs) { + entries.push({ question: p.question, answer: p.answer }) } } + for (const key of Object.keys(responses)) { + if (!startsWithSPCPFieldTitle(key)) continue + const ndi = responses[key] + entries.push({ + question: key, + answer: (ndi.answer as { value: string }).value, + }) + } + return JSON.stringify(entries) } +/** + * Prepares responses data from MRF responses to PDF html format + * @param formFields - The form fields schema + * @param responses - The mrf responses to the form fields + * @returns list of EmailRespondentConfirmationField used for email & pdf generation + */ +export const getResponsesDataFromMrfResponses = ({ + formFields, + responses, +}: { + formFields: FormFieldSchema[] | FormFieldDto[] + responses: FieldResponsesV4 +}): EmailRespondentConfirmationField[] => { + if (!formFields || !responses) return [] + + const questionAnswerPairs = getQuestionAnswerPairsForMultipleFields({ + formFields, + responses, + includeSignatureDataPngDataUri: true, + }) + + return questionAnswerPairs.map((questionAnswerPair) => { + return { + question: questionAnswerPair.question, + answerTemplate: [questionAnswerPair.answer], + answer: questionAnswerPair.signatureDataPngDataUri, + fieldType: questionAnswerPair.fieldType, + } + }) +} + /** * Creates a MRF cookie signed by FormSG * @param prevSubmissionId of the submission (same across all steps) diff --git a/apps/backend/src/app/modules/submission/receiver/receiver.middleware.ts b/apps/backend/src/app/modules/submission/receiver/receiver.middleware.ts index 77141c6773..dbe99e7d22 100644 --- a/apps/backend/src/app/modules/submission/receiver/receiver.middleware.ts +++ b/apps/backend/src/app/modules/submission/receiver/receiver.middleware.ts @@ -1,10 +1,7 @@ +import type { FieldResponsesV4 } from '@opengovsg/formsg-sdk' import { Busboy } from 'busboy' import { NextFunction, Request, Response } from 'express-serve-static-core' -import { - FieldResponse, - FieldResponsesV3, - FormResponseMode, -} from 'formsg-shared/types' +import { FieldResponse, FormResponseMode } from 'formsg-shared/types' import { Result } from 'neverthrow' import { createLoggerWithLabel } from '../../../config/logger' @@ -82,7 +79,7 @@ export const receiveStorageSubmission: ControllerHandler< export const receiveMultirespondentSubmission: ControllerHandler< unknown, { message: string }, - { responses: FieldResponsesV3 } + { responses: FieldResponsesV4 } > = async (req, res, next) => { return receiveSubmission( req, @@ -99,7 +96,7 @@ const receiveSubmission = async ( req: Request< unknown, { message: string }, - { responses: FieldResponse[] | FieldResponsesV3 }, + { responses: FieldResponse[] | FieldResponsesV4 }, unknown, Record >, diff --git a/apps/backend/src/app/modules/submission/receiver/receiver.service.ts b/apps/backend/src/app/modules/submission/receiver/receiver.service.ts index 86f131277a..24468cbc97 100644 --- a/apps/backend/src/app/modules/submission/receiver/receiver.service.ts +++ b/apps/backend/src/app/modules/submission/receiver/receiver.service.ts @@ -1,10 +1,7 @@ +import type { FieldResponsesV4 } from '@opengovsg/formsg-sdk' import Busboy from 'busboy' import { MB } from 'formsg-shared/constants/file' -import { - FieldResponse, - FieldResponsesV3, - FormResponseMode, -} from 'formsg-shared/types' +import { FieldResponse, FormResponseMode } from 'formsg-shared/types' import { IncomingHttpHeaders } from 'http' import { err, ok, Result, ResultAsync } from 'neverthrow' @@ -20,6 +17,7 @@ import { } from './receiver.errors' import { ParsedMultipartForm } from './receiver.types' import { + adaptMrfV3BodyToV4, addAttachmentToResponses, handleDuplicatesInAttachments, } from './receiver.utils' @@ -79,17 +77,17 @@ export const createMultipartReceiver = ( export const configureMultipartReceiver = ( busboy: Busboy.Busboy, ): ResultAsync< - ParsedMultipartForm, + ParsedMultipartForm, MultipartError > => { const logMeta = { action: 'configureMultipartReceiver', } const responsePromise = new Promise< - ParsedMultipartForm + ParsedMultipartForm >((resolve, reject) => { const attachments: IAttachmentInfo[] = [] - let body: ParsedMultipartForm + let body: ParsedMultipartForm busboy .on('file', (fieldname, file, { filename }) => { @@ -164,6 +162,7 @@ export const configureMultipartReceiver = ( }) .on('close', () => { if (body) { + adaptMrfV3BodyToV4(body) handleDuplicatesInAttachments(attachments) addAttachmentToResponses(body, attachments) return resolve(body) diff --git a/apps/backend/src/app/modules/submission/receiver/receiver.types.ts b/apps/backend/src/app/modules/submission/receiver/receiver.types.ts index 12c0ba1619..08efb94db4 100644 --- a/apps/backend/src/app/modules/submission/receiver/receiver.types.ts +++ b/apps/backend/src/app/modules/submission/receiver/receiver.types.ts @@ -1,8 +1,5 @@ -import { - FieldResponse, - FieldResponsesV3, - ResponseMetadata, -} from 'formsg-shared/types' +import type { FieldResponsesV4 } from '@opengovsg/formsg-sdk' +import { FieldResponse, ResponseMetadata } from 'formsg-shared/types' export type ParsedMultipartForm = { responses: ResponsesType @@ -24,6 +21,6 @@ export const isBodyVersion2AndBelow = ( */ export const isBodyVersion3AndAbove = ( body: ParsedMultipartForm, -): body is ParsedMultipartForm => { +): body is ParsedMultipartForm => { return (body.version ?? 0) >= 3 } diff --git a/apps/backend/src/app/modules/submission/receiver/receiver.utils.ts b/apps/backend/src/app/modules/submission/receiver/receiver.utils.ts index f9cb39da00..6aa8b5f6c9 100644 --- a/apps/backend/src/app/modules/submission/receiver/receiver.utils.ts +++ b/apps/backend/src/app/modules/submission/receiver/receiver.utils.ts @@ -1,16 +1,14 @@ +import type { FieldResponsesV4, FormFieldsV3 } from '@opengovsg/formsg-sdk' +import { adaptV3ToV4 } from '@opengovsg/formsg-sdk' import { VIRUS_SCANNER_SUBMISSION_VERSION } from 'formsg-shared/constants' -import { - BasicField, - FieldResponse, - FieldResponsesV3, -} from 'formsg-shared/types' +import { BasicField, FieldResponse } from 'formsg-shared/types' import { StatusCodes } from 'http-status-codes' import { IAttachmentInfo, MapRouteError } from '../../../../types' import { + ParsedClearAttachmentFieldResponseV4, ParsedClearAttachmentResponse, ParsedClearFormFieldResponse, - ParsedClearFormFieldResponseV3, } from '../../../../types/api' import { createLoggerWithLabel } from '../../../config/logger' @@ -53,6 +51,41 @@ export const mapRouteError: MapRouteError = (error) => { } } +/** + * Stale-FE compatibility shim for MRF submissions. + * + * If body.version indicates a V3 MRF submission (>=3 && <4), responses are V3-shaped. + * Convert them to V4 in place and bump body.version to 4 so downstream code can + * uniformly treat the body as V4. Runs BEFORE addAttachmentToResponses, so + * attachment buffers land in V4-shaped answer objects. + * + * Question text is left empty here — the form definition isn't yet loaded. Downstream + * code that needs question text should source it from the form definition. + */ +export const adaptMrfV3BodyToV4 = ( + body: ParsedMultipartForm, +): void => { + const version = body.version ?? 0 + if ( + version < 3 || + version >= 4 || + !body.responses || + Array.isArray(body.responses) + ) { + return + } + + logger.warn({ + message: + 'Adapting V3 MRF submission to V4 — client is on a stale build and should refresh', + meta: { action: 'adaptMrfV3BodyToV4', version }, + }) + + body.responses = adaptV3ToV4( + body.responses as unknown as FormFieldsV3, + ) as FieldResponsesV4 +} + /** * Adds the attachment's content, filename to each response, * based on their fieldId. @@ -63,7 +96,7 @@ export const mapRouteError: MapRouteError = (error) => { * @returns void. Modifies responses in place. */ export const addAttachmentToResponses = ( - body: ParsedMultipartForm, + body: ParsedMultipartForm, attachments: IAttachmentInfo[], ): void => { // default to 0 for email mode forms where version is undefined @@ -103,7 +136,9 @@ export const addAttachmentToResponses = ( if (isBodyVersion3AndAbove(body)) { Object.keys(body.responses).forEach((id) => { - const response = body.responses[id] as ParsedClearFormFieldResponseV3 + const response = body.responses[ + id + ] as unknown as ParsedClearAttachmentFieldResponseV4 if (response.fieldType === BasicField.Attachment && id in attachmentMap) { const file = attachmentMap[id] response.answer.filename = file.filename diff --git a/apps/backend/src/app/modules/submission/submission.service.ts b/apps/backend/src/app/modules/submission/submission.service.ts index 9d46170d1d..c09638034a 100644 --- a/apps/backend/src/app/modules/submission/submission.service.ts +++ b/apps/backend/src/app/modules/submission/submission.service.ts @@ -33,6 +33,7 @@ import { } from '../../../types' import { ParsedClearAttachmentFieldResponseV3, + ParsedClearAttachmentFieldResponseV4, ParsedClearAttachmentResponse, ParsedClearFormFieldResponse, } from '../../../types/api' @@ -1299,3 +1300,79 @@ export const triggerGuardDutyScanThenDownloadCleanFileChain = < }) ) } + +/** + * V4 version: Helper function to trigger guardduty scanning and download clean file. + * In V4, attachment data is nested inside response.answer rather than at the top level. + * @param response quarantined V4 attachment response + * @returns modified response with answer.content replaced with clean file buffer, + * and answer.value/answer.filename set to the real (uploaded) filename. + */ +export const triggerGuardDutyScanThenDownloadCleanFileChainV4 = ( + response: ParsedClearAttachmentFieldResponseV4, + formId: string, +): ResultAsync< + ParsedClearAttachmentFieldResponseV4, + TriggerGuardDutyScanThenDownloadCleanFileChainError +> => { + const quarantineFileKey = response.answer.value + const logMeta = { + action: 'triggerGuardDutyScanThenDownloadCleanFileChainV4', + formId, + quarantineFileKey, + } + return triggerGuardDutyScanning(quarantineFileKey) + .mapErr((error) => { + if (error instanceof GuardDutyMaliciousFileDetectedError) { + logger.error({ + message: 'GUARDDUTY Malicious file detected during lambda virus scan', + meta: logMeta, + error, + }) + return new GuardDutyMaliciousFileDetectedError(response.answer.filename) + } + return error + }) + .map((lambdaOutput) => { + logger.info({ + message: + 'GUARDDUTY Successfully retrieved clean file from virus scanning lambda', + meta: { ...logMeta, cleanFileKey: lambdaOutput.body.cleanFileKey }, + }) + return lambdaOutput.body + }) + .andThen((cleanAttachment) => + downloadCleanFile( + cleanAttachment.cleanFileKey, + cleanAttachment.destinationVersionId, + AwsConfig.guarddutyCleanS3Bucket, + ).map( + (attachmentBuffer) => + ({ + ...response, + answer: { + ...response.answer, + content: attachmentBuffer, + filename: response.answer.filename, + // Promote the real filename into `value` (the canonical, durable + // answer field). Until now `value` held the quarantine bucket key + // (a bare UUID); every downstream sink — email/PDF Q&A, response + // JSON, the V4->V3 webhook adapter, the encrypted-at-rest answer, + // and the admin CSV/attachment download name — reads `value`, so + // leaving it as the UUID drops the filename (and its extension). + // This mirrors the V3 chain's `answer: response.filename`. + value: response.answer.filename, + }, + }) as ParsedClearAttachmentFieldResponseV4, + ), + ) + .mapErr((error) => { + if (error instanceof DownloadCleanFileFailedError) { + return new GuardDutyDownloadCleanFileFailedError() + } + if (error instanceof InvalidFileKeyError) { + return new GuardDutyInvalidFileKeyError() + } + return error + }) +} diff --git a/apps/backend/src/app/utils/field-validation/validators/__tests__/email-validation.spec.ts b/apps/backend/src/app/utils/field-validation/validators/__tests__/email-validation.spec.ts index b298b98cd0..d5d01e4ad4 100644 --- a/apps/backend/src/app/utils/field-validation/validators/__tests__/email-validation.spec.ts +++ b/apps/backend/src/app/utils/field-validation/validators/__tests__/email-validation.spec.ts @@ -1,13 +1,22 @@ import { + generateDefaultField, generateDefaultFieldV3, generateVerifiableAnswerResponseV3, } from '__tests__/unit/backend/helpers/generate-form-data' import { BasicField } from 'formsg-shared/types' import formsgSdk from 'src/app/config/formsg-sdk' -import { ValidateFieldError } from 'src/app/modules/submission/submission.errors' +import { + ValidateFieldError, + ValidateFieldErrorV4, +} from 'src/app/modules/submission/submission.errors' import { ProcessedFieldResponse } from 'src/app/modules/submission/submission.types' -import { validateField, validateFieldV3 } from 'src/app/utils/field-validation' +import { + validateField, + validateFieldV3, + validateFieldV4, +} from 'src/app/utils/field-validation' +import { ParsedClearFormFieldResponseV4 } from 'src/types/api' import { FieldValidationSchema, IEmailFieldSchema, @@ -1158,3 +1167,182 @@ describe('Email field validation V3', () => { ) }) }) + +describe('Email field validation V4', () => { + beforeEach(() => { + jest + .spyOn( + formsgSdk.verification as unknown as VerificationMock, + 'authenticate', + ) + .mockImplementation(() => true) + }) + + const makeEmailResponseV4 = (answer: { + value: string + signature?: string + }): ParsedClearFormFieldResponseV4 => + ({ + fieldType: BasicField.Email, + question: 'Email', + answer, + provenance: {}, + }) as ParsedClearFormFieldResponseV4 + + it('should allow valid emails', () => { + const formField = generateDefaultField(BasicField.Email) + const response = makeEmailResponseV4({ value: 'valid@email.com' }) + const validateResult = validateFieldV4({ + formId: 'formId', + formField, + response, + isVisible: true, + }) + expect(validateResult.isOk()).toBe(true) + expect(validateResult._unsafeUnwrap()).toEqual(true) + }) + + it('should disallow invalid emails', () => { + const formField = generateDefaultField(BasicField.Email) + const response = makeEmailResponseV4({ value: 'invalidemail.com' }) + const validateResult = validateFieldV4({ + formId: 'formId', + formField, + response, + isVisible: true, + }) + expect(validateResult.isErr()).toBe(true) + expect(validateResult._unsafeUnwrapErr()).toEqual( + new ValidateFieldErrorV4('Invalid answer submitted'), + ) + }) + + it('should allow email addresses whose email domain belongs to allowedEmailDomains', () => { + const formField = generateDefaultField(BasicField.Email, { + isVerifiable: false, + hasAllowedEmailDomains: true, + allowedEmailDomains: ['@test.gov.sg'], + }) + const response = makeEmailResponseV4({ + value: 'volunteer-testing@test.gov.sg', + }) + const validateResult = validateFieldV4({ + formId: 'formId', + formField, + response, + isVisible: true, + }) + expect(validateResult.isOk()).toBe(true) + expect(validateResult._unsafeUnwrap()).toEqual(true) + }) + + it('should allow email addresses supplied with a mixed-case domain', () => { + const formField = generateDefaultField(BasicField.Email, { + isVerifiable: false, + hasAllowedEmailDomains: true, + allowedEmailDomains: ['@test.gov.sg'], // note: domains are always read lowercased from store + }) + const response = makeEmailResponseV4({ + value: 'volunteer-testing@TeSt.GoV.Sg', // mixed case domain + }) + const validateResult = validateFieldV4({ + formId: 'formId', + formField, + response, + isVisible: true, + }) + expect(validateResult.isOk()).toBe(true) + expect(validateResult._unsafeUnwrap()).toEqual(true) + }) + + // Regression: V4 previously used exact string equality and rejected valid + // wildcard-domain submissions that V3 accepted. See emailValidator.ts. + it('should allow email addresses matching a wildcard domain pattern (V4)', () => { + const formField = generateDefaultField(BasicField.Email, { + isVerifiable: false, + hasAllowedEmailDomains: true, + allowedEmailDomains: ['@*.moe.gov.sg'], + }) + const response = makeEmailResponseV4({ value: 'user@mail.moe.gov.sg' }) + const validateResult = validateFieldV4({ + formId: 'formId', + formField, + response, + isVisible: true, + }) + expect(validateResult.isOk()).toBe(true) + expect(validateResult._unsafeUnwrap()).toEqual(true) + }) + + it('should allow email addresses matching a multi-level wildcard subdomain (V4)', () => { + const formField = generateDefaultField(BasicField.Email, { + isVerifiable: false, + hasAllowedEmailDomains: true, + allowedEmailDomains: ['@*.moe.gov.sg'], + }) + const response = makeEmailResponseV4({ value: 'user@dept.mail.moe.gov.sg' }) + const validateResult = validateFieldV4({ + formId: 'formId', + formField, + response, + isVisible: true, + }) + expect(validateResult.isOk()).toBe(true) + expect(validateResult._unsafeUnwrap()).toEqual(true) + }) + + it('should not allow the base domain itself against a wildcard pattern (V4)', () => { + const formField = generateDefaultField(BasicField.Email, { + isVerifiable: false, + hasAllowedEmailDomains: true, + allowedEmailDomains: ['@*.moe.gov.sg'], + }) + const response = makeEmailResponseV4({ value: 'user@moe.gov.sg' }) + const validateResult = validateFieldV4({ + formId: 'formId', + formField, + response, + isVisible: true, + }) + expect(validateResult.isErr()).toBe(true) + expect(validateResult._unsafeUnwrapErr()).toEqual( + new ValidateFieldErrorV4('Invalid answer submitted'), + ) + }) + + it('should not allow email addresses whose email domain does not belong to allowedEmailDomains', () => { + const formField = generateDefaultField(BasicField.Email, { + isVerifiable: false, + hasAllowedEmailDomains: true, + allowedEmailDomains: ['@example.com'], + }) + const response = makeEmailResponseV4({ value: 'user@test.gov.sg' }) + const validateResult = validateFieldV4({ + formId: 'formId', + formField, + response, + isVisible: true, + }) + expect(validateResult.isErr()).toBe(true) + expect(validateResult._unsafeUnwrapErr()).toEqual( + new ValidateFieldErrorV4('Invalid answer submitted'), + ) + }) + + it('should reject email addresses if isVerifiable is true but there is no signature present', () => { + const formField = generateDefaultField(BasicField.Email, { + isVerifiable: true, + }) + const response = makeEmailResponseV4({ value: 'valid@email.com' }) + const validateResult = validateFieldV4({ + formId: 'formId', + formField, + response, + isVisible: true, + }) + expect(validateResult.isErr()).toBe(true) + expect(validateResult._unsafeUnwrapErr()).toEqual( + new ValidateFieldErrorV4('Invalid answer submitted'), + ) + }) +}) diff --git a/apps/backend/src/app/utils/field-validation/validators/emailValidator.ts b/apps/backend/src/app/utils/field-validation/validators/emailValidator.ts index 1f31b70752..e7c952112d 100644 --- a/apps/backend/src/app/utils/field-validation/validators/emailValidator.ts +++ b/apps/backend/src/app/utils/field-validation/validators/emailValidator.ts @@ -176,10 +176,10 @@ const makeEmailDomainValidatorV4: ResponseValidatorConstructor< const emailAddress = String(value).trim() if (!(hasAllowedEmailDomains && allowedEmailDomains.length)) return right(response) - const emailDomain = ('@' + emailAddress.split('@').pop()).toLowerCase() + const emailDomain = '@' + emailAddress.split('@').pop() - return allowedEmailDomains.some( - (domain) => domain.toLowerCase() === emailDomain, + return allowedEmailDomains.some((domain) => + emailDomainMatchesAllowed(emailDomain, domain), ) ? right(response) : left(`EmailValidatorV4:\t answer value is not a valid email domain`) diff --git a/apps/backend/src/types/api/multirespondent_submission.ts b/apps/backend/src/types/api/multirespondent_submission.ts index 09e48758f3..c776728cd7 100644 --- a/apps/backend/src/types/api/multirespondent_submission.ts +++ b/apps/backend/src/types/api/multirespondent_submission.ts @@ -1,5 +1,5 @@ +import type { FieldResponsesV4 } from '@opengovsg/formsg-sdk' import { - FieldResponsesV3, FormFieldDto, FormLogic, FormResponseMode, @@ -11,7 +11,7 @@ import { IPopulatedMultirespondentForm } from '../form' import { IMultirespondentSubmissionSchema } from '../submission' export type ParsedMultirespondentSubmissionBody = { - responses: FieldResponsesV3 + responses: FieldResponsesV4 responseMetadata?: ResponseMetadata version: number workflowStep: number @@ -63,6 +63,6 @@ export type MultirespondentSubmissionDto = { workflowStep: number hashedSubmitterId?: string submitterId?: string - responses: FieldResponsesV3 + responses: FieldResponsesV4 mrfVersion: number } diff --git a/packages/sdk/cjs-entry.cjs b/packages/sdk/cjs-entry.cjs index c5e9617a24..e00fe8bbbb 100644 --- a/packages/sdk/cjs-entry.cjs +++ b/packages/sdk/cjs-entry.cjs @@ -2,4 +2,7 @@ // TypeScript compiles `export default function` to `exports.default = fn`, // but CJS consumers expect `require('@opengovsg/formsg-sdk')` to return // the function directly (matching the old v0.15.0 behavior). -module.exports = require('./dist/cjs/index.js').default +// Object.assign preserves named exports (adaptV3ToV4, etc.) on the +// callable default so both `formsg(config)` and `formsg.adaptV3ToV4` work. +const mod = require('./dist/cjs/index.js') +module.exports = Object.assign(mod.default, mod)