Merge pull request #57 from doug-martin/master

v0.4.4

History.md

@@ -1,6 +1,10 @@
+# 0.4.4
+
+* Added support for comments. [#56](https://github.com/C2FO/fast-csv/issues/56)
+
 # v0.4.3
 
-* Added ability to include a `rowDelimiter` at the end of a csv with the `includeEndRowDelimiter` optioin [#54](https://github.com/C2FO/fast-csv/issues/54)
+* Added ability to include a `rowDelimiter` at the end of a csv with the `includeEndRowDelimiter` option [#54](https://github.com/C2FO/fast-csv/issues/54)
 * Added escaping for values that include a row delimiter
 * Added more tests for new feature and escaping row delimiter values.
 

README.md

@@ -28,6 +28,7 @@ All methods accept the following `options`
   * `trim=false`: If you want to trim all values parsed set to true.
   * `rtrim=false`: If you want to right trim all values parsed set to true.
   * `ltrim=false`: If you want to left trim all values parsed set to true.
+  * `comment=null`: If your CSV contains comments you can use this option to ignore lines that begin with the specified character (e.g. `#`).
 
 
 **events**

docs/History.html

@@ -176,9 +176,13 @@
 
 
 
+<h1>0.4.4</h1>
+<ul>
+<li>Added support for comments. <a href="https://github.com/C2FO/fast-csv/issues/56">#56</a></li>
+</ul>
 <h1>v0.4.3</h1>
 <ul>
-<li>Added ability to include a <code>rowDelimiter</code> at the end of a csv with the <code>includeEndRowDelimiter</code> optioin <a href="https://github.com/C2FO/fast-csv/issues/54">#54</a></li>
+<li>Added ability to include a <code>rowDelimiter</code> at the end of a csv with the <code>includeEndRowDelimiter</code> option <a href="https://github.com/C2FO/fast-csv/issues/54">#54</a></li>
 <li>Added escaping for values that include a row delimiter</li>
 <li>Added more tests for new feature and escaping row delimiter values.</li>
 </ul>

docs/index.html

@@ -204,6 +204,7 @@ <h3>Parsing</h3>
 <li><code>trim=false</code>: If you want to trim all values parsed set to true.</li>
 <li><code>rtrim=false</code>: If you want to right trim all values parsed set to true.</li>
 <li><code>ltrim=false</code>: If you want to left trim all values parsed set to true.</li>
+<li><code>comment=null</code>: If your CSV contains comments you can use this option to ignore lines that begin with the specified character (e.g. <code>#</code>).</li>
 </ul>
 </li>
 </ul>

lib/parser.js

@@ -1,4 +1,5 @@
 var extended = require("./extended"),
+    has = extended.has,
     isUndefinedOrNull = extended.isUndefinedOrNull,
     trim = extended.trim,
     trimLeft = extended.trimLeft,
@@ -15,7 +16,12 @@ function createParser(options) {
         SEARCH_REGEXP = new RegExp("(?:\\n|\\r|" + delimiter + ")"),
         ESCAPE_CHAR = options.escape || '"',
         NEXT_TOKEN_REGEXP = new RegExp("([^\\s]|\\r\\n|\\n|\\r|" + delimiter + ")"),
-        LINE_BREAK = /(\r\n|\n|\r)/;
+        ROW_DELIMITER = /(\r\n|\n|\r)/,
+        COMMENT, hasComments;
+    if (has(options, "comment")) {
+        COMMENT = options.comment;
+        hasComments = true;
+    }
 
     function formatItem(item) {
         if (doTrim) {
@@ -71,7 +77,7 @@ function createParser(options) {
             }
         } else if ((!depth && nextToken && nextToken.search(SEARCH_REGEXP) === -1)) {
             throw new Error("Parse Error: expected: '" + ESCAPE + "' got: '" + nextToken + "'. at '" + str.substr(cursor, 10).replace(/[\r\n]/g, "\\n" + "'"));
-        } else if (hasMoreData && (!nextToken || !LINE_BREAK.test(nextToken))) {
+        } else if (hasMoreData && (!nextToken || !ROW_DELIMITER.test(nextToken))) {
             cursor = null;
         }
         if (cursor !== null) {
@@ -80,6 +86,20 @@ function createParser(options) {
         return cursor;
     }
 
+    function parseCommentLine(line, cursor, hasMoreData) {
+        var nextIndex = line.substr(cursor).search(ROW_DELIMITER);
+        if (nextIndex === -1) {
+            if (hasMoreData) {
+                nextIndex = null;
+            } else {
+                nextIndex = line.length + 1;
+            }
+        } else {
+            nextIndex = (cursor + nextIndex) + 1; //go past the next line break
+        }
+        return nextIndex;
+    }
+
     function parseItem(line, items, cursor, hasMoreData) {
         var searchStr = line.substr(cursor),
             nextIndex = searchStr.search(SEARCH_REGEXP);
@@ -98,7 +118,7 @@ function createParser(options) {
                 items.push(formatItem(searchStr.substr(0, nextIndex)));
                 cursor += nextIndex + 1;
             }
-        } else if (LINE_BREAK.test(nextChar)) {
+        } else if (ROW_DELIMITER.test(nextChar)) {
             items.push(formatItem(searchStr.substr(0, nextIndex)));
             cursor += nextIndex;
         } else if (!hasMoreData) {
@@ -128,7 +148,7 @@ function createParser(options) {
             if (isUndefinedOrNull(token)) {
                 i = lastLineI;
                 break;
-            } else if (LINE_BREAK.test(token)) {
+            } else if (ROW_DELIMITER.test(token)) {
                 i = nextToken.cursor + 1;
                 if (i < l) {
                     rows.push(items);
@@ -137,6 +157,18 @@ function createParser(options) {
                 } else {
                     break;
                 }
+            } else if (hasComments && token === COMMENT) {
+                cursor = parseCommentLine(line, i, hasMoreData);
+                if (cursor === null) {
+                    i = lastLineI;
+                    break;
+                } else if (cursor < l) {
+                    lastLineI = i = cursor;
+                } else {
+                    i = cursor;
+                    cursor = null;
+                    break;
+                }
             } else {
                 if (token === ESCAPE) {
                     cursor = parseEscapedItem(line, items, nextToken.cursor, hasMoreData);

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fast-csv",
-  "version": "0.4.3",
+  "version": "0.4.4",
   "description": "CSV parser and writer",
   "main": "index.js",
   "scripts": {

test/assets/test24.csv

@@ -0,0 +1,23 @@
+#This is a test CSV
+#It contains a bunch of comments!!!!
+#The fist row contains headers.
+first_name,last_name,email_address,address
+#Line 1
+First1,Last1,[email protected],"1 Street St, State ST, 88888"
+#Line 2
+First2,Last2,[email protected],"2 Street St, State ST, 88888"
+#Line 3
+First3,Last3,[email protected],"3 Street St, State ST, 88888"
+#Line 4
+First4,Last4,[email protected],"4 Street St, State ST, 88888"
+#Line 5
+First5,Last5,[email protected],"5 Street St, State ST, 88888"
+#Line 6
+First6,Last6,[email protected],"6 Street St, State ST, 88888"
+#Line 7
+First7,Last7,[email protected],"7 Street St, State ST, 88888"
+#Line 8
+First8,Last8,[email protected],"8 Street St, State ST, 88888"
+#Line 9
+First9,Last9,[email protected],"9 Street St, State ST, 88888"
+#End of CSV

test/fast-csv.test.js

@@ -825,6 +825,20 @@ it.describe("fast-csv", function (it) {
             });
     });
 
+    it.should("handle CSVs with comments", function (next) {
+        var actual = [];
+        csv
+            .fromPath(path.resolve(__dirname, "./assets/test24.csv"), {headers: true, comment: "#"})
+            .on("record", function (data, index) {
+                actual[index] = data;
+            }).
+            on("end", function (count) {
+                assert.deepEqual(actual, expected1);
+                assert.equal(count, actual.length);
+                next();
+            });
+    });
+
     it.describe("pause/resume", function () {
 
         it.should("support pausing a stream", function (next) {