Merge pull request #268 from mrodrig/fix-265

Fix 265
mrodrig · Nov 24, 2024 · 2b9f6e3 · 2b9f6e3
2 parents 9551140 + 2155500
commit 2b9f6e3
Show file tree

Hide file tree

Showing 8 changed files with 113 additions and 63 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -21,6 +21,7 @@
         "compile": "tsc -p tsconfig.build.json",
         "coverage": "nyc npm run test",
         "lint": "eslint --ext .js,.ts src test",
+        "lint:fix": "eslint --fix --ext .js,.ts src test",
         "prepublishOnly": "npm run build",
         "test": "mocha -r ts-node/register test/index.ts"
     },
@@ -59,4 +60,4 @@
         "node": ">= 16"
     },
     "license": "MIT"
-}
+}
diff --git a/src/csv2json.ts b/src/csv2json.ts
@@ -5,7 +5,7 @@ import { excelBOM } from './constants';
 import type { Csv2JsonParams, FullCsv2JsonOptions, HeaderField } from './types';
 import * as utils from './utils';
 
-export const Csv2Json = function(options: FullCsv2JsonOptions) {
+export const Csv2Json = function (options: FullCsv2JsonOptions) {
     const escapedWrapDelimiterRegex = new RegExp(options.delimiter.wrap + options.delimiter.wrap, 'g'),
         excelBOMRegex = new RegExp('^' + excelBOM),
         valueParserFn = options.parseValue && typeof options.parseValue === 'function' ? options.parseValue : JSON.parse;
@@ -166,9 +166,18 @@ export const Csv2Json = function(options: FullCsv2JsonOptions) {
                 if (utils.getNCharacters(csv, index + 1, eolDelimiterLength) === options.delimiter.eol) {
                     index += options.delimiter.eol.length + 1; // Skip past EOL
                 }
-            }
-
-            else if ((charBefore !== options.delimiter.wrap || stateVariables.justParsedDoubleQuote && charBefore === options.delimiter.wrap) &&
+            } else if (charBefore === options.delimiter.field && character === options.delimiter.wrap && charAfter === options.delimiter.eol) {
+                // We reached the start of a wrapped new field that begins with an EOL delimiter
+
+                // Retrieve the remaining value and add it to the split line list of values
+                splitLine.push(csv.substring(stateVariables.startIndex, index - 1));
+
+                stateVariables.startIndex = index;
+                stateVariables.parsingValue = true;
+                stateVariables.insideWrapDelimiter = true;
+                stateVariables.justParsedDoubleQuote = true;
+                index += 1;
+            } else if ((charBefore !== options.delimiter.wrap || stateVariables.justParsedDoubleQuote && charBefore === options.delimiter.wrap) &&
                 character === options.delimiter.wrap && utils.getNCharacters(csv, index + 1, eolDelimiterLength) === options.delimiter.eol) {
                 // If we reach a wrap which is not preceded by a wrap delim and the next character is an EOL delim (ie. *"\n)
 

diff --git a/test/config/testCsvFilesList.ts b/test/config/testCsvFilesList.ts
@@ -2,60 +2,61 @@ import { readFileSync } from 'fs';
 import path from 'path';
 
 const csvFileConfig = [
-    {key: 'noData', file: '../data/csv/noData.csv'},
-    {key: 'singleDocument', file: '../data/csv/singleDocument.csv'},
-    {key: 'array', file: '../data/csv/array.csv'},
-    {key: 'arrayObjects', file: '../data/csv/arrayObjects.csv'},
-    {key: 'arrayMixedObjNonObj', file: '../data/csv/arrayMixedObjNonObj.csv'},
-    {key: 'arraySingleArray', file: '../data/csv/arraySingleArray.csv'},
-    {key: 'date', file: '../data/csv/date.csv'},
-    {key: 'null', file: '../data/csv/null.csv'},
-    {key: 'undefined', file: '../data/csv/undefined.csv'},
-    {key: 'nested', file: '../data/csv/nested.csv'},
-    {key: 'nestedMissingField', file: '../data/csv/nestedMissingField.csv'},
-    {key: 'comma', file: '../data/csv/comma.csv'},
-    {key: 'quotes', file: '../data/csv/quotes.csv'},
-    {key: 'quotesHeader', file: '../data/csv/quotesHeader.csv'},
-    {key: 'quotesAndCommas', file: '../data/csv/quotesAndCommas.csv'},
-    {key: 'eol', file: '../data/csv/eol.csv'},
-    {key: 'assortedValues', file: '../data/csv/assortedValues.csv'},
-    {key: 'trimFields', file: '../data/csv/trimFields.csv'},
-    {key: 'trimmedFields', file: '../data/csv/trimmedFields.csv'},
-    {key: 'trimHeader', file: '../data/csv/trimHeader.csv'},
-    {key: 'trimmedHeader', file: '../data/csv/trimmedHeader.csv'},
-    {key: 'excelBOM', file: '../data/csv/excelBOM.csv'},
-    {key: 'specifiedKeys', file: '../data/csv/specifiedKeys.csv'},
-    {key: 'specifiedKeysNoData', file: '../data/csv/specifiedKeysNoData.csv'},
-    {key: 'extraLine', file: '../data/csv/extraLine.csv'},
-    {key: 'noHeader', file: '../data/csv/noHeader.csv'},
-    {key: 'sortedHeader', file: '../data/csv/sortedHeader.csv'},
-    {key: 'sortedHeaderCustom', file: '../data/csv/sortedHeaderCustom.csv'},
-    {key: 'emptyFieldValues', file: '../data/csv/emptyFieldValues.csv'},
-    {key: 'quotedEmptyFieldValue', file: '../data/csv/quotedEmptyFieldValue.csv'},
-    {key: 'csvEmptyLastValue', file: '../data/csv/csvEmptyLastValue.csv'},
-    {key: 'unwind', file: '../data/csv/unwind.csv'},
-    {key: 'unwindEmptyArray', file: '../data/csv/unwindEmptyArray.csv'},
-    {key: 'unwindWithSpecifiedKeys', file: '../data/csv/unwindWithSpecifiedKeys.csv'},
-    {key: 'withSpecifiedKeys', file: '../data/csv/withSpecifiedKeys.csv'},
-    {key: 'localeFormat', file: '../data/csv/localeFormat.csv'},
-    {key: 'invalidParsedValues', file: '../data/csv/invalidParsedValues.csv'},
-    {key: 'firstColumnWrapCRLF', file: '../data/csv/firstColumnWrapCRLF.csv'},
-    {key: 'emptyLastFieldValue', file: '../data/csv/emptyLastFieldValue.csv'},
-    {key: 'emptyLastFieldValueNoEol', file: '../data/csv/emptyLastFieldValueNoEol.csv'},
-    {key: 'lastCharFieldDelimiter', file: '../data/csv/lastCharFieldDelimiter.csv'},
-    {key: 'nativeMapMethod', file: '../data/csv/nativeMapMethod.csv'},
-    {key: 'nestedDotKeys', file: '../data/csv/nestedDotKeys.csv'},
-    {key: 'nestedDotKeysWithArray', file: '../data/csv/nestedDotKeysWithArray.csv'},
-    {key: 'nestedDotKeysWithArrayExpandedUnwound', file: '../data/csv/nestedDotKeysWithArrayExpandedUnwound.csv'},
-    {key: 'emptyColumns', file: '../data/csv/emptyColumns.csv'},
-    {key: 'quotedFieldWithNewline', file: '../data/csv/quotedFieldWithNewline.csv'},
-    {key: 'falsyValues', file: '../data/csv/falsyValues.csv'},
-    {key: 'nestedNotUnwoundObjects', file: '../data/csv/nestedNotUnwoundObjects.csv'},
-    {key: 'newlineWithWrapDelimiters', file: '../data/csv/newlineWithWrapDelimiters.csv'},
-    {key: 'excludeKeyPattern', file: '../data/csv/excludeKeyPattern.csv'},
-    {key: 'wildcardMatch', file: '../data/csv/wildcardMatch.csv'},
-    {key: 'arrayIndexesAsKeys', file: '../data/csv/arrayIndexesAsKeys.csv'},
-    {key: 'keyWithEndingDot', file:'../data/csv/keyWithEndingDot.csv'},
+    { key: 'noData', file: '../data/csv/noData.csv' },
+    { key: 'singleDocument', file: '../data/csv/singleDocument.csv' },
+    { key: 'array', file: '../data/csv/array.csv' },
+    { key: 'arrayObjects', file: '../data/csv/arrayObjects.csv' },
+    { key: 'arrayMixedObjNonObj', file: '../data/csv/arrayMixedObjNonObj.csv' },
+    { key: 'arraySingleArray', file: '../data/csv/arraySingleArray.csv' },
+    { key: 'date', file: '../data/csv/date.csv' },
+    { key: 'null', file: '../data/csv/null.csv' },
+    { key: 'undefined', file: '../data/csv/undefined.csv' },
+    { key: 'nested', file: '../data/csv/nested.csv' },
+    { key: 'nestedMissingField', file: '../data/csv/nestedMissingField.csv' },
+    { key: 'comma', file: '../data/csv/comma.csv' },
+    { key: 'quotes', file: '../data/csv/quotes.csv' },
+    { key: 'quotesHeader', file: '../data/csv/quotesHeader.csv' },
+    { key: 'quotesAndCommas', file: '../data/csv/quotesAndCommas.csv' },
+    { key: 'eol', file: '../data/csv/eol.csv' },
+    { key: 'assortedValues', file: '../data/csv/assortedValues.csv' },
+    { key: 'trimFields', file: '../data/csv/trimFields.csv' },
+    { key: 'trimmedFields', file: '../data/csv/trimmedFields.csv' },
+    { key: 'trimHeader', file: '../data/csv/trimHeader.csv' },
+    { key: 'trimmedHeader', file: '../data/csv/trimmedHeader.csv' },
+    { key: 'excelBOM', file: '../data/csv/excelBOM.csv' },
+    { key: 'specifiedKeys', file: '../data/csv/specifiedKeys.csv' },
+    { key: 'specifiedKeysNoData', file: '../data/csv/specifiedKeysNoData.csv' },
+    { key: 'extraLine', file: '../data/csv/extraLine.csv' },
+    { key: 'noHeader', file: '../data/csv/noHeader.csv' },
+    { key: 'sortedHeader', file: '../data/csv/sortedHeader.csv' },
+    { key: 'sortedHeaderCustom', file: '../data/csv/sortedHeaderCustom.csv' },
+    { key: 'emptyFieldValues', file: '../data/csv/emptyFieldValues.csv' },
+    { key: 'quotedEmptyFieldValue', file: '../data/csv/quotedEmptyFieldValue.csv' },
+    { key: 'csvEmptyLastValue', file: '../data/csv/csvEmptyLastValue.csv' },
+    { key: 'unwind', file: '../data/csv/unwind.csv' },
+    { key: 'unwindEmptyArray', file: '../data/csv/unwindEmptyArray.csv' },
+    { key: 'unwindWithSpecifiedKeys', file: '../data/csv/unwindWithSpecifiedKeys.csv' },
+    { key: 'withSpecifiedKeys', file: '../data/csv/withSpecifiedKeys.csv' },
+    { key: 'localeFormat', file: '../data/csv/localeFormat.csv' },
+    { key: 'invalidParsedValues', file: '../data/csv/invalidParsedValues.csv' },
+    { key: 'firstColumnWrapCRLF', file: '../data/csv/firstColumnWrapCRLF.csv' },
+    { key: 'emptyLastFieldValue', file: '../data/csv/emptyLastFieldValue.csv' },
+    { key: 'emptyLastFieldValueNoEol', file: '../data/csv/emptyLastFieldValueNoEol.csv' },
+    { key: 'lastCharFieldDelimiter', file: '../data/csv/lastCharFieldDelimiter.csv' },
+    { key: 'nativeMapMethod', file: '../data/csv/nativeMapMethod.csv' },
+    { key: 'nestedDotKeys', file: '../data/csv/nestedDotKeys.csv' },
+    { key: 'nestedDotKeysWithArray', file: '../data/csv/nestedDotKeysWithArray.csv' },
+    { key: 'nestedDotKeysWithArrayExpandedUnwound', file: '../data/csv/nestedDotKeysWithArrayExpandedUnwound.csv' },
+    { key: 'emptyColumns', file: '../data/csv/emptyColumns.csv' },
+    { key: 'quotedFieldWithNewline', file: '../data/csv/quotedFieldWithNewline.csv' },
+    { key: 'falsyValues', file: '../data/csv/falsyValues.csv' },
+    { key: 'nestedNotUnwoundObjects', file: '../data/csv/nestedNotUnwoundObjects.csv' },
+    { key: 'newlineWithWrapDelimiters', file: '../data/csv/newlineWithWrapDelimiters.csv' },
+    { key: 'excludeKeyPattern', file: '../data/csv/excludeKeyPattern.csv' },
+    { key: 'wildcardMatch', file: '../data/csv/wildcardMatch.csv' },
+    { key: 'arrayIndexesAsKeys', file: '../data/csv/arrayIndexesAsKeys.csv' },
+    { key: 'keyWithEndingDot', file: '../data/csv/keyWithEndingDot.csv' },
+    { key: 'fieldEolAtStart', file: '../data/csv/fieldEolAtStart.csv' },
 ];
 
 function readCsvFile(filePath: string) {

diff --git a/test/config/testJsonFilesList.ts b/test/config/testJsonFilesList.ts
@@ -48,4 +48,5 @@ export default {
     wildcardMatch: require('../data/json/wildcardMatch.json'),
     arrayIndexesAsKeys: require('../data/json/arrayIndexesAsKeys.json'),
     keyWithEndingDot: require('../data/json/keyWithEndingDot.json'),
+    fieldEolAtStart: require('../data/json/fieldEolAtStart.json'),
 };
diff --git a/test/csv2json.ts b/test/csv2json.ts
@@ -62,7 +62,7 @@ export function runTests() {
 
             it('should convert csv representing nested json with a missing field to the correct json structure', () => {
                 const json = csv2json(csvTestData.nestedMissingField);
-                jsonTestData.nestedMissingField[0].level1.level2.level3 = {level: null};
+                jsonTestData.nestedMissingField[0].level1.level2.level3 = { level: null };
                 assert.deepEqual(json, jsonTestData.nestedMissingField);
             });
 
@@ -177,6 +177,12 @@ export function runTests() {
                 const json = csv2json(csvTestData.keyWithEndingDot);
                 assert.deepEqual(json, jsonTestData.keyWithEndingDot);
             });
+
+            // Test case for #265
+            it('should handle fields starting with an EOL delimiter', () => {
+                const json = csv2json(csvTestData.fieldEolAtStart);
+                assert.deepEqual(json, jsonTestData.fieldEolAtStart);
+            });
         });
 
         describe('Error Handling', () => {

diff --git a/test/data/csv/fieldEolAtStart.csv b/test/data/csv/fieldEolAtStart.csv
@@ -0,0 +1,6 @@
+id,name,age,city
+1,John Doe,29,New York
+2,Jane Smith,34,"
+ Los Angeles"
+3,Emily Davis,22,Chicago
+4,Michael Brown,40,Houston
diff --git a/test/data/json/fieldEolAtStart.json b/test/data/json/fieldEolAtStart.json
@@ -0,0 +1,26 @@
+[
+    {
+        "id": 1,
+        "name": "John Doe",
+        "age": 29,
+        "city": "New York"
+    },
+    {
+        "id": 2,
+        "name": "Jane Smith",
+        "age": 34,
+        "city": "\n Los Angeles"
+    },
+    {
+        "id": 3,
+        "name": "Emily Davis",
+        "age": 22,
+        "city": "Chicago"
+    },
+    {
+        "id": 4,
+        "name": "Michael Brown",
+        "age": 40,
+        "city": "Houston"
+    }
+]