From 7793620a57b07cfc88c7d3dc4b06bd7dbb942e8b Mon Sep 17 00:00:00 2001 From: Doreen Schwartz <126075185+Doreen-Schwartz@users.noreply.github.com> Date: Tue, 10 Sep 2024 16:40:53 +0300 Subject: [PATCH] Alternative Field Encoding for Types (#33) * feat: encode + decode with alternative field codec --- src/__snapshots__/nistEncode.test.ts.snap | 177 ++++++++++++++++++++++ src/nistDecode.test.ts | 70 +++++++++ src/nistDecode.ts | 36 ++++- src/nistEncode.test.ts | 38 +++++ src/nistEncode.ts | 65 +++++--- src/nistVisitor.ts | 12 +- 6 files changed, 369 insertions(+), 29 deletions(-) diff --git a/src/__snapshots__/nistEncode.test.ts.snap b/src/__snapshots__/nistEncode.test.ts.snap index aa629ef..40bc05f 100644 --- a/src/__snapshots__/nistEncode.test.ts.snap +++ b/src/__snapshots__/nistEncode.test.ts.snap @@ -439,6 +439,183 @@ exports[`positive test: Type-1 and Type-2 only with default options - encode int } `; +exports[`positive test: Type-1, Type-2 with alternative informationWriter 1`] = ` +{ + "data": [ + 49, + 46, + 48, + 48, + 49, + 58, + 49, + 49, + 51, + 29, + 49, + 46, + 48, + 48, + 50, + 58, + 48, + 53, + 48, + 50, + 29, + 49, + 46, + 48, + 48, + 51, + 58, + 49, + 31, + 49, + 30, + 50, + 31, + 48, + 48, + 29, + 49, + 46, + 48, + 48, + 52, + 58, + 67, + 82, + 77, + 29, + 49, + 46, + 48, + 48, + 53, + 58, + 50, + 48, + 49, + 57, + 49, + 50, + 48, + 49, + 29, + 49, + 46, + 48, + 48, + 55, + 58, + 68, + 65, + 73, + 48, + 51, + 53, + 52, + 53, + 52, + 29, + 49, + 46, + 48, + 48, + 56, + 58, + 79, + 82, + 73, + 51, + 56, + 53, + 55, + 52, + 51, + 53, + 52, + 29, + 49, + 46, + 48, + 48, + 57, + 58, + 84, + 67, + 78, + 50, + 52, + 56, + 55, + 83, + 48, + 53, + 52, + 28, + 50, + 46, + 48, + 48, + 49, + 58, + 53, + 54, + 29, + 50, + 46, + 48, + 48, + 50, + 58, + 48, + 48, + 29, + 50, + 46, + 48, + 48, + 52, + 58, + 74, + 111, + 104, + 110, + 29, + 50, + 46, + 48, + 48, + 53, + 58, + 68, + 111, + 101, + 29, + 50, + 46, + 48, + 48, + 55, + 58, + 49, + 57, + 55, + 56, + 45, + 48, + 53, + 45, + 49, + 50, + 28, + ], + "type": "Buffer", +} +`; + exports[`positive test: Type-1, Type-2, Type-4 with an overflow for 2.001 (record length) 1`] = ` { "data": [ diff --git a/src/nistDecode.test.ts b/src/nistDecode.test.ts index 804e78f..ce2ded9 100644 --- a/src/nistDecode.test.ts +++ b/src/nistDecode.test.ts @@ -560,6 +560,76 @@ describe('positive test:', () => { ], }); }); + + it('decode field 2.005 with alternative information decoder', () => { + const nist: NistFile = { + 1: { + 2: '0502', + 4: 'CRM', + 5: '20191201', + 7: 'DAI035454', + 8: 'ORI38574354', + 9: 'TCN2487S054', + }, + 2: { + 4: 'John', + 5: 'Doe', + 7: '1978-05-12', + }, + }; + + const buffer = nistEncode(nist, { + codecOptions: { + default: { + 2: { + 5: { + informationWriter: (information) => { + if (typeof information == 'string') return Buffer.from(information, 'latin1'); + return information; + }, + }, + }, + }, + }, + }); + expect(buffer.tag).toEqual('success'); + + const result = nistDecode((buffer as Success).value, { + codecOptions: { + default: { + 2: { + 5: { + informationDecoder: (buffer) => buffer.toString('latin1'), + }, + }, + }, + }, + }); + + expect(result.tag).toEqual('success'); + expect((result as Success).value).toEqual({ + 1: { + 1: '113', + 2: '0502', + 3: [ + ['1', '1'], + ['2', '00'], + ], + 4: 'CRM', + 5: '20191201', + 7: 'DAI035454', + 8: 'ORI38574354', + 9: 'TCN2487S054', + }, + 2: { + 1: '56', + 2: '00', + 4: 'John', + 5: 'Doe', + 7: '1978-05-12', + }, + }); + }); }); describe('negative test:', () => { diff --git a/src/nistDecode.ts b/src/nistDecode.ts index fc0b3c3..9f5ed26 100644 --- a/src/nistDecode.ts +++ b/src/nistDecode.ts @@ -26,6 +26,7 @@ import { SEPARATOR_UNIT, } from './nistUtils'; import { nistValidation } from './nistValidation'; +import { getPerTotOptions } from './nistVisitor'; import { failure, Result, success } from './result'; /** Decoding options for a single NIST Field. */ @@ -38,6 +39,7 @@ export interface NistFieldDecodeOptions extends NistFieldCodecOptions { * This behaviour can be overridden on a per-field basis by passing a custom parser property. */ parser?: (field: NistField, nist: NistFile) => Result; + informationDecoder?: (buffer: Buffer, startOffset: number, endOffset: number) => string; } /** Decoding options for one NIST record. */ @@ -130,7 +132,9 @@ const decodeNistSubfield = ( buffer: Buffer, startOffset: number, endOffset: number, + options?: NistFieldDecodeOptions, ): NistSubfield => { + const decoder = options?.informationDecoder || stringValue; let offset = startOffset; let unitSeparator = findSeparator(buffer, SEPARATOR_UNIT, offset, endOffset); @@ -139,7 +143,7 @@ const decodeNistSubfield = ( while (offset <= endOffset) { subfield = [ ...subfield, - stringValue(buffer, offset, unitSeparator ? unitSeparator - 1 : endOffset), + decoder(buffer, offset, unitSeparator ? unitSeparator - 1 : endOffset), ]; offset = (unitSeparator || endOffset) + 1; unitSeparator = findSeparator(buffer, SEPARATOR_UNIT, offset, endOffset); @@ -149,9 +153,9 @@ const decodeNistSubfield = ( // The same logic is present also in decodeNistFieldValue. if (alwaysDecodeAsSet(key)) { - return [stringValue(buffer, startOffset, endOffset)]; + return [decoder(buffer, startOffset, endOffset)]; } - return stringValue(buffer, startOffset, endOffset); + return decoder(buffer, startOffset, endOffset); }; const decodeNistFieldValue = ( @@ -159,7 +163,9 @@ const decodeNistFieldValue = ( buffer: Buffer, startOffset: number, endOffset: number, + options?: NistFieldDecodeOptions, ): NistFieldValue => { + const decoder = options?.informationDecoder || stringValue; let offset = startOffset; let recordSeparator = findSeparator(buffer, SEPARATOR_RECORD, offset, endOffset); @@ -170,9 +176,9 @@ const decodeNistFieldValue = ( if (!unitSeparator) { // The same logic is present also in decodeNistSubfield. if (alwaysDecodeAsSet(key)) { - return [[stringValue(buffer, startOffset, endOffset)]]; + return [[decoder(buffer, startOffset, endOffset)]]; } - return stringValue(buffer, startOffset, endOffset); + return decoder(buffer, startOffset, endOffset); } // Also deal with the case there is no record separator but some unit separators. return [decodeNistSubfield(key, buffer, startOffset, endOffset)]; @@ -343,6 +349,7 @@ export const decodeGenericNistRecord = ( recordInstance: number, startOffset: number, endOffset: number, + options?: NistRecordDecodeOptions, ): Result => { let offset = startOffset; let recordEndOffset: number | null = null; // This will be assigned after parsing LEN field. @@ -388,7 +395,13 @@ export const decodeGenericNistRecord = ( const value = nistFieldKey.value.key.field === 999 ? buffer.subarray(valueStartOffset, fieldEndOffset) - : decodeNistFieldValue(nistFieldKey.value.key, buffer, valueStartOffset, fieldEndOffset); + : decodeNistFieldValue( + nistFieldKey.value.key, + buffer, + valueStartOffset, + fieldEndOffset, + options, + ); const nistField = { key: nistFieldKey.value.key, value }; if (nistField.key.field === 1) { @@ -445,7 +458,10 @@ const addRecord = ( } }; -const decodeNistFile = (buffer: Buffer): Result => { +const decodeNistFile = ( + buffer: Buffer, + options: NistDecodeOptions, +): Result => { let offset = 0; const endOffset = buffer.length - 1; @@ -477,6 +493,9 @@ const decodeNistFile = (buffer: Buffer): Result => { // 1. Decode the buffer. - const nistFileInternal = decodeNistFile(buffer); + const nistFileInternal = decodeNistFile(buffer, options); if (nistFileInternal.tag === 'failure') { return nistFileInternal; } diff --git a/src/nistEncode.test.ts b/src/nistEncode.test.ts index 35398ed..c029481 100644 --- a/src/nistEncode.test.ts +++ b/src/nistEncode.test.ts @@ -596,6 +596,44 @@ describe('positive test:', () => { expect((result as Success).value.byteLength).toBe(880293); expect((result as Success).value).toMatchSnapshot(); }); + + it('Type-1, Type-2 with alternative informationWriter', () => { + const nist: NistFile = { + 1: { + 2: '0502', + 4: 'CRM', + 5: '20191201', + 7: 'DAI035454', + 8: 'ORI38574354', + 9: 'TCN2487S054', + }, + 2: { + 4: 'John', + 5: 'Doe', + 7: '1978-05-12', + }, + }; + + const result = nistEncode(nist, { + ...nistEncodeOptions, + codecOptions: { + ...nistEncodeOptions.codecOptions, + default: { + 2: { + 5: { + informationWriter: (information) => { + if (typeof information == 'string') return Buffer.from(information, 'latin1'); + return information; + }, + }, + }, + }, + }, + }); + expect(result.tag).toEqual('success'); + expect((result as Success).value.byteLength).toBe(169); + expect((result as Success).value).toMatchSnapshot(); + }); }); describe('negative test:', () => { diff --git a/src/nistEncode.ts b/src/nistEncode.ts index 15d328c..326849f 100644 --- a/src/nistEncode.ts +++ b/src/nistEncode.ts @@ -36,6 +36,7 @@ import { failure, Result, success } from './result'; /** Encoding options for a single NIST Field. */ interface NistFieldEncodeOptions extends NistFieldCodecOptions { formatter?: (field: NistField, nist: NistFile) => NistFieldValue; + informationWriter?: (informationItem: NistInformationItem) => Buffer | undefined; } /** Encoding options for one NIST record. */ @@ -79,6 +80,10 @@ const invokeFormatters = ({ return success(undefined); }; +const defaultInformationWriter = (informationItem: string): Buffer => { + return Buffer.from(informationItem); +}; + /* --------------------------- Automatic fields ------------------------------------------------- */ const determineCharset = ({ nist }: { nist: NistFile }): Result => { @@ -122,49 +127,54 @@ interface LengthTracking { totalLength: number; } -const informationItemLength = (informationItem: NistInformationItem): number => +const informationItemLength = ( + informationItem: NistInformationItem, + options?: NistFieldEncodeOptions, +): number => informationItem ? typeof informationItem === 'string' - ? Buffer.byteLength(informationItem) // utf-8 is the default + ? options?.informationWriter?.(informationItem)?.byteLength ?? + defaultInformationWriter(informationItem).byteLength // utf-8 is the default : informationItem.byteLength : 0; -const subfieldLength = (subfield: NistSubfield): number => +const subfieldLength = (subfield: NistSubfield, options?: NistFieldEncodeOptions): number => Array.isArray(subfield) ? subfield.reduce( - (total, informationItem) => total + informationItemLength(informationItem), + (total, informationItem) => total + informationItemLength(informationItem, options), 0, ) + (subfield.length - 1) // unit separators - : informationItemLength(subfield); + : informationItemLength(subfield, options); -const fieldValueLength = (value: NistFieldValue): number => +const fieldValueLength = (value: NistFieldValue, options?: NistFieldEncodeOptions): number => Array.isArray(value) ? (value as NistInformationItem[]).reduce( - (total, subfield) => total + subfieldLength(subfield), + (total, subfield) => total + subfieldLength(subfield, options), 0, ) + (value.length - 1) // record separators - : subfieldLength(value); + : subfieldLength(value, options); -const computeFieldLength = (field: NistField): number => { +const computeFieldLength = (field: NistField, options?: NistFieldEncodeOptions): number => { const fieldNumberLength = formatFieldKey(field.key.type, field.key.field).length; - const valueLength = fieldValueLength(field.value); + const valueLength = fieldValueLength(field.value, options); return fieldNumberLength + 1 + valueLength + 1; // 1 for ':', 1 for group separator }; const assignFieldLength: NistFieldVisitorFn = ({ field, data, + options, }): NistFieldVisitorFnReturn => { - data.recordLength += computeFieldLength(field); + data.recordLength += computeFieldLength(field, options); return success(undefined); }; const assignRecordLength: NistRecordVisitorFn = ( params, ): Result => { - const { nist, recordTypeNumber, record, recordNumber, visitorStrategy, data } = params; + const { nist, recordTypeNumber, record, recordNumber, visitorStrategy, data, options } = params; let recordLength; if (recordTypeNumber === 4) { @@ -180,6 +190,7 @@ const assignRecordLength: NistRecordVisitorFn => { const tracking = { currentIdc: 0, contentRecordCount: 0, records: [] }; // 1. assign IDCs to xx.002 and determine value for 1.003 @@ -232,6 +245,7 @@ const computeAutomaticFields = ({ nist, recordVisitor: { fn: assignRecordLength, data: lengthTracking }, visitorStrategy: {}, + options: options.codecOptions, }); if (result.tag === 'failure') { return result; @@ -350,44 +364,52 @@ const encodeType4Record = ( const encodeNistInformationItem = ( informationItem: NistInformationItem, data: EncodeTracking, + options?: NistFieldEncodeOptions, ): void => { if (informationItem) { if (typeof informationItem === 'string') { - data.offset += data.buf.write(informationItem, data.offset); // utf-8 is the default + const encodedBuffer = + options?.informationWriter?.(informationItem) ?? defaultInformationWriter(informationItem); + data.offset += encodedBuffer.copy(data.buf, data.offset); } else { data.offset += informationItem.copy(data.buf, data.offset); } } }; -const encodeNistSubfield = (subfield: NistSubfield, data: EncodeTracking): void => { +const encodeNistSubfield = ( + subfield: NistSubfield, + data: EncodeTracking, + options?: NistFieldEncodeOptions, +): void => { if (Array.isArray(subfield)) { subfield.forEach((informationItem, index, array) => { - encodeNistInformationItem(informationItem, data); + encodeNistInformationItem(informationItem, data, options); if (index < array.length - 1) { data.offset = data.buf.writeUInt8(SEPARATOR_UNIT, data.offset); } }); } else { - encodeNistInformationItem(subfield, data); + encodeNistInformationItem(subfield, data, options); } }; const encodeNistField: NistFieldVisitorFn = ({ field, data, + options, }): NistFieldVisitorFnReturn => { data.offset += data.buf.write(formatFieldKey(field.key.type, field.key.field), data.offset); data.offset = data.buf.writeUInt8(SEPARATOR_FIELD_NUMBER, data.offset); if (Array.isArray(field.value)) { field.value.forEach((subfield, index, array) => { - encodeNistSubfield(subfield, data); + encodeNistSubfield(subfield, data, options); if (index < array.length - 1) { data.offset = data.buf.writeUInt8(SEPARATOR_RECORD, data.offset); } }); } else { - encodeNistSubfield(field.value, data); + encodeNistSubfield(field.value, data, options); } data.offset = data.buf.writeUInt8(SEPARATOR_GROUP, data.offset); @@ -416,9 +438,11 @@ const encodeNistRecord: NistRecordVisitorFn => { const encodeTracking = { buf, offset: 0 }; return visitNistFile({ @@ -426,6 +450,7 @@ const encodeNistFile = ({ nist, recordVisitor: { fn: encodeNistRecord, data: encodeTracking }, visitorStrategy: {}, + options: options?.codecOptions, }); }; @@ -453,7 +478,7 @@ export const nistPopulate = ( determineCharset({ nist }); // 4. compute automatic fields: xx.002 (IDC), 1.003 and 1.015, xx.001 - const result = computeAutomaticFields({ nist }); + const result = computeAutomaticFields({ nist, options }); if (result.tag === 'failure') { return result; } @@ -491,7 +516,7 @@ export const nistEncode = ( } // 4. encode all fields into the buf (including arrays of subfields) - const result3 = encodeNistFile({ nist: nistPopulated, buf }); + const result3 = encodeNistFile({ nist: nistPopulated, buf, options }); if (result3.tag === 'failure') { return result3; } diff --git a/src/nistVisitor.ts b/src/nistVisitor.ts index b20c39f..e29507b 100644 --- a/src/nistVisitor.ts +++ b/src/nistVisitor.ts @@ -8,6 +8,7 @@ import { NistFieldValue, NistFile, NistFileCodecOptions, + NistFileCodecOptionsPerTot, NistRecord, NistRecordCodecOptions, } from './index'; @@ -216,6 +217,15 @@ interface VisitNistFileParams< recordVisitor?: NistRecordVisitor; fieldVisitor?: NistFieldVisitor; } + +export const getPerTotOptions = < + T extends NistFieldCodecOptions, + U extends NistRecordCodecOptions, +>( + options: NistFileCodecOptions, + tot: string, +): NistFileCodecOptionsPerTot => R.mergeDeepRight(options.default, options[tot] || {}); + /* Visits the whole NistFile, all types, all records, all fields. recordVisitor and fieldVisitor can be overriden. */ export const visitNistFile = < @@ -232,7 +242,7 @@ export const visitNistFile = < const nistFile = nist as unknown as NistFileInternal; let summaryResult: Result = success(undefined); // assume success - const perTotOptions = options && R.mergeDeepRight(options.default, options[nist[1][4]] || {}); + const perTotOptions = options && getPerTotOptions(options, nist[1][4]); for (const recordTypeNumber of nistRecordTypeNumbers) { if (nistFile[recordTypeNumber]) {