From 287542d2d82e56e034194cbdc3cefea9b82b8a93 Mon Sep 17 00:00:00 2001 From: Liam Mitchell Date: Sat, 8 Apr 2023 06:14:59 +1200 Subject: [PATCH 1/3] Add skipEmptyLines and extendedRangeError Proposing a solution for #193 As well as a way to skip empty lines Or better yet could extract logic from writeRow to map cells to object and return that as the row? --- index.js | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/index.js b/index.js index 2a6a2da..30c5b91 100644 --- a/index.js +++ b/index.js @@ -159,11 +159,21 @@ class CsvParser extends Transform { return } - if (!skip && this.options.strict && cells.length !== this.headers.length) { - const e = new RangeError('Row length does not match headers') - this.emit('error', e) - } else { - if (!skip) this.writeRow(cells) + if (!skip) { + if (start === end && this.options.skipEmptyLines) { + return + } + if (this.options.strict && cells.length !== this.headers.length) { + const e = new RangeError('Row length does not match headers') + if (this.options.extendedRangeError) { + e.lineNumber = this.state.lineNumber + e.cells = cells + e.line = buffer.toString('utf-8', start, end) + } + this.emit('error', e) + } else { + this.writeRow(cells) + } } } From c5a5ba619c2138c285b0c16c29b76aa7e6e0ac45 Mon Sep 17 00:00:00 2001 From: Liam Date: Sat, 8 Apr 2023 07:28:52 +1200 Subject: [PATCH 2/3] test: skipemptylines docs: skipemptylines --- README.md | 8 ++++++++ bin/csv-parser | 4 +++- index.d.ts | 7 +++++++ index.test-d.ts | 1 + test/fixtures/option-skipEmptyLines.csv | 7 +++++++ test/skipEmptyLines.test.js | 17 +++++++++++++++++ 6 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 test/fixtures/option-skipEmptyLines.csv create mode 100644 test/skipEmptyLines.test.js diff --git a/README.md b/README.md index bfd4b31..2632649 100644 --- a/README.md +++ b/README.md @@ -241,6 +241,13 @@ Default: `0` Specifies the number of lines at the beginning of a data file that the parser should skip over, prior to parsing headers. +#### skipEmptyLines + +Type: `Boolean`
+Default: `false` + +If `true`, empty lines are skipped. + #### maxRowBytes Type: `Number`
@@ -305,6 +312,7 @@ Usage: csv-parser [filename?] [options] --separator,-s Set the separator character ("," by default) --skipComments,-c Skip CSV comments that begin with '#'. Set a value to change the comment character. --skipLines,-l Set the number of lines to skip to before parsing headers + --skipEmptyLines Skip empty lines --strict Require column length match headers length --version,-v Print out the installed version ``` diff --git a/bin/csv-parser b/bin/csv-parser index f895a8e..f3b23fb 100755 --- a/bin/csv-parser +++ b/bin/csv-parser @@ -44,6 +44,7 @@ if (argv.help || (process.stdin.isTTY && !filename)) { --separator,-s Set the separator character ("," by default) --skipComments,-c Skip CSV comments that begin with '#'. Set a value to change the comment character. --skipLines,-l Set the number of lines to skip to before parsing headers + --skipEmptyLines Skip empty lines --strict Require column length match headers length --version,-v Print out the installed version `) @@ -56,7 +57,8 @@ const options = { separator: argv.separator, strict: argv.strict, skipComments: argv.skipComments, - skipLines: argv.skipLines + skipLines: argv.skipLines, + skipEmptyLines: argv.skipEmptyLines } if (argv.headers) { diff --git a/index.d.ts b/index.d.ts index a6a400d..9186a44 100644 --- a/index.d.ts +++ b/index.d.ts @@ -96,6 +96,13 @@ Bugs Bunny,22 */ readonly skipLines?: number; + /** + * If true empty lines are skipped. + * + * @default false + */ + readonly skipEmptyLines?: boolean; + /** * Maximum number of bytes per row. An error is thrown if a line exeeds this value. The default value is on 8 peta byte. * diff --git a/index.test-d.ts b/index.test-d.ts index 703330e..b11fd58 100644 --- a/index.test-d.ts +++ b/index.test-d.ts @@ -36,5 +36,6 @@ expectType(csvParser({ separator: ',' })); expectType(csvParser({ skipComments: true })); expectType(csvParser({ skipComments: '#' })); expectType(csvParser({ skipLines: 1 })); +expectType(csvParser({ skipEmptyLines: false })); expectType(csvParser({ maxRowBytes: 1 })); expectType(csvParser({ strict: true })); diff --git a/test/fixtures/option-skipEmptyLines.csv b/test/fixtures/option-skipEmptyLines.csv new file mode 100644 index 0000000..96083f6 --- /dev/null +++ b/test/fixtures/option-skipEmptyLines.csv @@ -0,0 +1,7 @@ +A + +1 +2 + + +3 diff --git a/test/skipEmptyLines.test.js b/test/skipEmptyLines.test.js new file mode 100644 index 0000000..90ab778 --- /dev/null +++ b/test/skipEmptyLines.test.js @@ -0,0 +1,17 @@ +const test = require('ava') + +const { collect } = require('./helpers/helper') + +test.cb('skip empty lines', (t) => { + const verify = (err, lines) => { + console.log(lines) + t.false(err, 'no err') + t.is(lines.length, 3, '3 row') + t.is(JSON.stringify(lines[0]), JSON.stringify({ A: '1' })) + t.is(JSON.stringify(lines[1]), JSON.stringify({ A: '2' })) + t.is(JSON.stringify(lines[2]), JSON.stringify({ A: '3' })) + t.end() + } + + collect('option-skipEmptyLines', { skipEmptyLines: true, columns: true, strict: true }, verify) +}) From e5e0f9968e7b01dfb1cea4fdb66b9e427c444e8a Mon Sep 17 00:00:00 2001 From: Liam Date: Sat, 8 Apr 2023 07:53:30 +1200 Subject: [PATCH 3/3] test: extendedrangeeror docs: extendedrangerror --- README.md | 7 +++++++ index.d.ts | 5 +++++ index.js | 6 +++--- index.test-d.ts | 1 + test/extendedRangeError.test.js | 14 ++++++++++++++ test/fixtures/option-extendedRangeError.csv | 2 ++ 6 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 test/extendedRangeError.test.js create mode 100644 test/fixtures/option-extendedRangeError.csv diff --git a/README.md b/README.md index 2632649..64eca77 100644 --- a/README.md +++ b/README.md @@ -266,6 +266,13 @@ if `false`: the headers are mapped to the column index less columns: any missing column in the middle will result in a wrong property mapping! more columns: the aditional columns will create a "_"+index properties - eg. "_10":"value" +#### extendedRangeError + +Type: `Boolean`
+Default: `false` + +If `true`, then `lineNumber`, `cells`, `line` are added to the error object. + ## Events The following events are emitted during parsing: diff --git a/index.d.ts b/index.d.ts index 9186a44..ad2f529 100644 --- a/index.d.ts +++ b/index.d.ts @@ -114,6 +114,11 @@ Bugs Bunny,22 * If `true`, instructs the parser that the number of columns in each row must match the number of `headers` specified. */ readonly strict?: boolean; + + /** + * If `true`, then `lineNumber`, `cells`, `line` are added to the error object. + */ + readonly extendedRangeError?: boolean; } } diff --git a/index.js b/index.js index 30c5b91..7b41d9a 100644 --- a/index.js +++ b/index.js @@ -166,9 +166,9 @@ class CsvParser extends Transform { if (this.options.strict && cells.length !== this.headers.length) { const e = new RangeError('Row length does not match headers') if (this.options.extendedRangeError) { - e.lineNumber = this.state.lineNumber - e.cells = cells - e.line = buffer.toString('utf-8', start, end) + e.lineNumber = this.state.lineNumber + e.cells = cells + e.line = buffer.toString('utf-8', start, end) } this.emit('error', e) } else { diff --git a/index.test-d.ts b/index.test-d.ts index b11fd58..9b46bd3 100644 --- a/index.test-d.ts +++ b/index.test-d.ts @@ -39,3 +39,4 @@ expectType(csvParser({ skipLines: 1 })); expectType(csvParser({ skipEmptyLines: false })); expectType(csvParser({ maxRowBytes: 1 })); expectType(csvParser({ strict: true })); +expectType(csvParser({ extendedRangeError: false })); diff --git a/test/extendedRangeError.test.js b/test/extendedRangeError.test.js new file mode 100644 index 0000000..c02b28c --- /dev/null +++ b/test/extendedRangeError.test.js @@ -0,0 +1,14 @@ +const test = require('ava') + +const { collect } = require('./helpers/helper') + +test.cb('extended range error', (t) => { + const verify = (err, lines) => { + t.is(err.lineNumber, 2, 'lineNumber set') + t.is(err.cells.length, 1, 'cells returned') + t.is(err.line, '1', 'line returned') + t.end() + } + + collect('option-extendedRangeError', { columns: true, strict: true, extendedRangeError: true }, verify) +}) diff --git a/test/fixtures/option-extendedRangeError.csv b/test/fixtures/option-extendedRangeError.csv new file mode 100644 index 0000000..1fc698d --- /dev/null +++ b/test/fixtures/option-extendedRangeError.csv @@ -0,0 +1,2 @@ +A,B +1