From 781fb2f8ab835aa1ca88158d1f1dad6811fa629b Mon Sep 17 00:00:00 2001 From: Daniel Edler Date: Wed, 10 Jan 2018 20:28:06 +0100 Subject: [PATCH 1/5] Add support and test for invalid encoding input to decoder-browser --- decoder-browser.js | 24 ++++++++++++++++++------ package.json | 17 ++++++++++------- test/basic.test.js | 29 +++++++++++++++++++++++++++-- test/data/encoding.cpg | 1 + test/index.html | 22 ++++++++++++++++++++++ 5 files changed, 78 insertions(+), 15 deletions(-) create mode 100644 test/data/encoding.cpg create mode 100644 test/index.html diff --git a/decoder-browser.js b/decoder-browser.js index 32f293d..a663831 100644 --- a/decoder-browser.js +++ b/decoder-browser.js @@ -1,4 +1,5 @@ require('text-encoding-polyfill'); +var iconv = require('iconv-lite'); var StringDecoder = require('string_decoder').StringDecoder; function defaultDecoder(data) { var decoder = new StringDecoder(); @@ -6,18 +7,29 @@ function defaultDecoder(data) { return out.replace(/\0/g, '').trim(); } module.exports = createDecoder; -var regex = /^(?:ASNI\s)?(\d+)$/m; +var regex = /^(?:ANSI\s)?(\d+)$/m; + function createDecoder(encoding) { if (!encoding) { return defaultDecoder; } + enconding = String(encoding).trim(); + var match = regex.exec(encoding); + if (match) { + encoding = 'windows-' + match[1]; + } + if (!iconv.encodingExists(encoding)) { + if (encoding.length > 5 && iconv.encodingExists(encoding.slice(5))) { + encoding = encoding.slice(5); + } else { + return defaultDecoder; + } + } try { - new TextDecoder(encoding.trim()); + new TextDecoder(encoding); + return browserDecoder; } catch(e) { - var match = regex.exec(encoding); - if (match) { - encoding = 'windows-' + match[1]; - } + return defaultDecoder; } return browserDecoder; function browserDecoder(buffer) { diff --git a/package.json b/package.json index 3cfbd34..0766d1a 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,7 @@ "description": "The DBF parsing component of shapefile-js", "main": "index.js", "scripts": { + "build-test": "browserify -t brfs ./test/basic.test.js > ./test/bundle.js", "test": "jshint index.js && mocha -R spec test/*.test.js" }, "repository": { @@ -20,15 +21,17 @@ "url": "https://github.com/calvinmetcalf/parseDBF/issues" }, "devDependencies": { - "chai": "^3.3.0", - "jshint": "^2.8.0", - "mocha": "^2.3.3" - }, - "browser": { - "./decoder.js": "./decoder-browser.js" + "brfs": "^1.4.3", + "browserify": "^15.0.0", + "chai": "^4.1.2", + "jshint": "^2.9.5", + "mocha": "^4.1.0" }, "dependencies": { - "iconv-lite": "^0.4.15", + "iconv-lite": "^0.4.19", "text-encoding-polyfill": "^0.6.7" + }, + "browser": { + "./decoder.js": "./decoder-browser.js" } } diff --git a/test/basic.test.js b/test/basic.test.js index ec83549..0cf570c 100644 --- a/test/basic.test.js +++ b/test/basic.test.js @@ -62,16 +62,41 @@ describe('dbf',function(){ done(); }); }); - it('should handle other charicters',function(done){ + it('should handle ANSI characters with different encoding names',function(done){ fs.readFile('./test/data/codepage.dbf',function(err,data){ if(err){ return done(err); } dbf(data)[1].should.not.deep.equal(utf[1]); dbf(data, '1250')[1].should.deep.equal(utf[1]); - dbf(data, 'ASNI 1250')[1].should.deep.equal(utf[1]); + dbf(data, 'ANSI 1250')[1].should.deep.equal(utf[1]); dbf(data, 'windows-1250')[1].should.deep.equal(utf[1]); done(); }); }); + it('should handle invalid encodings',function(done){ + var htmlpage = fs.readFileSync('./test/index.html', 'utf-8'); + fs.readFile('./test/data/watershed.dbf',function(err,data){ + if(err){ + return done(err); + } + // throws if calling data.trim() + dbf(data, data).should.deep.equal(basic); + // throws RangeError if using TextDecoder(htmlpage) + dbf(data, htmlpage).should.deep.equal(basic); + done(); + }); + }); + it('should handle non-decoded encodings',function(done){ + var encoding = fs.readFileSync('./test/data/encoding.cpg', 'utf-8'); + var encodingBinary = fs.readFileSync('./test/data/encoding.cpg'); + fs.readFile('./test/data/utf.dbf',function(err,data){ + if(err){ + return done(err); + } + dbf(data, encoding).should.deep.equal(utf); + dbf(data, encodingBinary).should.deep.equal(utf); + done(); + }); + }); }); diff --git a/test/data/encoding.cpg b/test/data/encoding.cpg new file mode 100644 index 0000000..b1759f7 --- /dev/null +++ b/test/data/encoding.cpg @@ -0,0 +1 @@ +utf8 \ No newline at end of file diff --git a/test/index.html b/test/index.html new file mode 100644 index 0000000..ad7239a --- /dev/null +++ b/test/index.html @@ -0,0 +1,22 @@ + + + + Mocha Tests + + + + + + + + +
+ + + From 99e96d07b6288f8540534ec0924d96c2e4089530 Mon Sep 17 00:00:00 2001 From: Daniel Edler Date: Wed, 10 Jan 2018 22:39:40 +0100 Subject: [PATCH 2/5] Use budo to build/serve/open/live-reload browser test --- index.html | 16 ++++++++++++++++ package.json | 4 ++-- test/index.html | 22 ---------------------- 3 files changed, 18 insertions(+), 24 deletions(-) create mode 100644 index.html delete mode 100644 test/index.html diff --git a/index.html b/index.html new file mode 100644 index 0000000..83340f5 --- /dev/null +++ b/index.html @@ -0,0 +1,16 @@ + + + + Mocha Tests + + + + + +
+ + + + diff --git a/package.json b/package.json index 0766d1a..daf1746 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "The DBF parsing component of shapefile-js", "main": "index.js", "scripts": { - "build-test": "browserify -t brfs ./test/basic.test.js > ./test/bundle.js", + "test-browser": "budo test/basic.test.js --serve test/bundle.js --open --live -- -t brfs", "test": "jshint index.js && mocha -R spec test/*.test.js" }, "repository": { @@ -22,7 +22,7 @@ }, "devDependencies": { "brfs": "^1.4.3", - "browserify": "^15.0.0", + "budo": "^10.0.4", "chai": "^4.1.2", "jshint": "^2.9.5", "mocha": "^4.1.0" diff --git a/test/index.html b/test/index.html deleted file mode 100644 index ad7239a..0000000 --- a/test/index.html +++ /dev/null @@ -1,22 +0,0 @@ - - - - Mocha Tests - - - - - - - - -
- - - From 93d21ed73ac6944015619aed30a965ec8d812415 Mon Sep 17 00:00:00 2001 From: Daniel Edler Date: Thu, 11 Jan 2018 17:21:31 +0100 Subject: [PATCH 3/5] Remove iconv-lite from browser implementation and fix test --- decoder-browser.js | 8 -------- test/basic.test.js | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/decoder-browser.js b/decoder-browser.js index a663831..e585b7c 100644 --- a/decoder-browser.js +++ b/decoder-browser.js @@ -1,5 +1,4 @@ require('text-encoding-polyfill'); -var iconv = require('iconv-lite'); var StringDecoder = require('string_decoder').StringDecoder; function defaultDecoder(data) { var decoder = new StringDecoder(); @@ -18,13 +17,6 @@ function createDecoder(encoding) { if (match) { encoding = 'windows-' + match[1]; } - if (!iconv.encodingExists(encoding)) { - if (encoding.length > 5 && iconv.encodingExists(encoding.slice(5))) { - encoding = encoding.slice(5); - } else { - return defaultDecoder; - } - } try { new TextDecoder(encoding); return browserDecoder; diff --git a/test/basic.test.js b/test/basic.test.js index 0cf570c..85c893d 100644 --- a/test/basic.test.js +++ b/test/basic.test.js @@ -75,7 +75,7 @@ describe('dbf',function(){ }); }); it('should handle invalid encodings',function(done){ - var htmlpage = fs.readFileSync('./test/index.html', 'utf-8'); + var htmlpage = fs.readFileSync('./index.html', 'utf-8'); fs.readFile('./test/data/watershed.dbf',function(err,data){ if(err){ return done(err); From eef62ab12e0d6b168b88567d10071ee775513e00 Mon Sep 17 00:00:00 2001 From: Daniel Edler Date: Thu, 11 Jan 2018 18:15:16 +0100 Subject: [PATCH 4/5] Fix failing tests for legacy node.js by downgrading mocha --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index daf1746..2b080ac 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "budo": "^10.0.4", "chai": "^4.1.2", "jshint": "^2.9.5", - "mocha": "^4.1.0" + "mocha": "^3.5.3" }, "dependencies": { "iconv-lite": "^0.4.19", From 312391376e618f215e76bbd35af18555ce8055b2 Mon Sep 17 00:00:00 2001 From: Daniel Edler Date: Thu, 11 Jan 2018 23:51:27 +0100 Subject: [PATCH 5/5] Fix typo --- decoder-browser.js | 2 +- test/basic.test.js | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/decoder-browser.js b/decoder-browser.js index e585b7c..0a57e91 100644 --- a/decoder-browser.js +++ b/decoder-browser.js @@ -12,7 +12,7 @@ function createDecoder(encoding) { if (!encoding) { return defaultDecoder; } - enconding = String(encoding).trim(); + encoding = String(encoding).trim(); var match = regex.exec(encoding); if (match) { encoding = 'windows-' + match[1]; diff --git a/test/basic.test.js b/test/basic.test.js index 85c893d..f05e199 100644 --- a/test/basic.test.js +++ b/test/basic.test.js @@ -80,9 +80,7 @@ describe('dbf',function(){ if(err){ return done(err); } - // throws if calling data.trim() dbf(data, data).should.deep.equal(basic); - // throws RangeError if using TextDecoder(htmlpage) dbf(data, htmlpage).should.deep.equal(basic); done(); });