From d79c3616e2f6e227d78de23bd625ed28fa87f92c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0tefan=20Baebler?= Date: Mon, 6 Apr 2015 22:15:20 +0200 Subject: [PATCH 1/4] Take up to 11 characters of field name from DBF DBF file structure allows up to 11 characters/bytes in the field name (http://en.wikipedia.org/wiki/.dbf#File_Architecture_Overview), while ShapeFile spec sets the limit to 10 (http://en.wikipedia.org/wiki/Shapefile ) within the same DBF. This change allows reading longer field names. Writing such files could be seen as producing non-standard compliant, but reading them does no harm (null characters are stripped out anyway). We're using this on http://raba.openstreetmap.si since https://github.com/openstreetmap-si/raba.openstreetmap.si/commit/b72c357ccc7d3926ed77acf2d97f8563f03b6bc6 (dbf files within shapefile zips were adjusted with [bbe](http://linux.die.net/man/1/bbe) to insert an extra character) --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 6a82245..bb7021a 100644 --- a/index.js +++ b/index.js @@ -14,7 +14,7 @@ function dbfRowHeader(buffer){ var offset = 32; while(true){ out.push({ - name : String.fromCharCode.apply(this,(new Uint8Array(buffer,offset,10))).replace(/\0|\s+$/g,''), + name : String.fromCharCode.apply(this,(new Uint8Array(buffer,offset,11))).replace(/\0|\s+$/g,''), dataType : String.fromCharCode(data.getUint8(offset+11)), len : data.getUint8(offset+16), decimal : data.getUint8(offset+17) From 0b42172713c4a232e5576b71bf215d44a0f2f595 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0tefan=20Baebler?= Date: Thu, 9 Apr 2015 00:00:21 +0200 Subject: [PATCH 2/4] unit test covering the 11 characters in the field name --- test/11characters.test.js | 23 +++++ test/data/watershed-11chars.dbf | Bin 0 -> 3626 bytes test/data/watershed-11chars.js | 165 ++++++++++++++++++++++++++++++++ 3 files changed, 188 insertions(+) create mode 100644 test/11characters.test.js create mode 100644 test/data/watershed-11chars.dbf create mode 100644 test/data/watershed-11chars.js diff --git a/test/11characters.test.js b/test/11characters.test.js new file mode 100644 index 0000000..22917fd --- /dev/null +++ b/test/11characters.test.js @@ -0,0 +1,23 @@ +var fs = require('fs'); +var out = require('./data/watershed-11chars'); +var dbf = require('../'); +require('chai').should(); +function toArrayBuffer(buffer) { + var ab = new ArrayBuffer(buffer.length); + var view = new Uint8Array(ab); + for (var i = 0; i < buffer.length; ++i) { + view[i] = buffer[i]; + } + return ab; +} +describe('basic',function(){ + it('should work',function(done){ + fs.readFile('./test/data/watershed-11chars.dbf',function(err,data){ + if(err){ + return done(err); + } + dbf(toArrayBuffer(data)).should.deep.equal(out); + done(); + }); + }); +}); diff --git a/test/data/watershed-11chars.dbf b/test/data/watershed-11chars.dbf new file mode 100644 index 0000000000000000000000000000000000000000..93bd9ddcd16834139ea77178525e36ffde55658d GIT binary patch literal 3626 zcmai1$!_CD5EYP966BOQ<4lS*O^U#oeEJuf+XX zKJWIw>F*cGGD|MfAVu~4o8Er8NUqZNNs%U#^SJf>w@+WF_g85)8Q%YnO}FpfCB?jC z)8uzH{0pXC|8TfIK3xCRPV1-SQV3F(6a~HiRfxbW_d+-sqz*E~CBtU8D1)g_zkMP4D||s(bcy8}&ePr*$Cfz3^2q0wXTHLTr>a!qqrgQZBYh@6)GKFWY(T zw|3S%!2?6u8RI$}#g%a6y*I*CCK?7t?#e})X1%XZrheRSDh6G9r=@CnAUTF!fdoo5 zd|O_XyX9wn4+uC~NfX&n1wF%x$JN(if zdGhoE;I#m?V=w}PiJ*qJ#!1!qZI_mt*ZB4tj@HgQK`!{}(7-eoLD?W= zFx2pEk#EanlV{6W@0yQO^%@UUFjm7+6e@B>dkk6Qy_L$eyj~>5DlH#ReLiwMdwO|_ zDlN1i%HLs^DXh8{s(&qrpiW#@ZbL*Y7!b=1w9_nEaIfM&Nv$l!P zc~Y#G*#&@J#L?q8;G0$gP9iY5#zE?maL_yn;i>I8FVbwB){J_7>s<(!Hww2$FkB^? zo&}D8f$gA&s+NeA`F67ayOVO0ta>Lj)6nV8iF!W1YPr%{dW6G>PDep%?Lle?c5TmZ z+hvwclkLUhhy4+a`%UxU3B-cP*~&>VPmO`>SRy4%jk8I%*;sG@YRCO1xpq<@cO)1W z4~_YQiljjc10FUwP`34Q^)lNSGq2$&6BLm)&e(WptaBtmHH1jjOT@CID3XgLE7Q%U zs@-ShA}`WY{V|=64#o#g63{W|wRIJVJ?b_hhJp$`3KcZ5Ww}Z+vL8?sDzvW@dFD(ldCR5(?9tPVm4?{$->h|MZ3 zM(xpEzcYvf-7&#v5g^V@e4y7Jgfq@bYWk3A9z};i<)(S@maV@WA8tRieyX3Y$LpTH zp&1AiVyyv{YaXCfkP|8(sp^tsSPy5n|MK_Y^TQ4MneFzs_Z5)q@^rAyDKt0AArcmL z)i|&Ru?i@4w$0NXPPYGaxI6A2PAzgB-v)6|Y?MMP;^3!7cVHsAIf+6MLZjDNUc9P1 z9nK4UBNEUyuMm^k9^$cvTj)b0d4^pj#bsKYUjNxRjK@cz??vXw0KYYQjebX?8I7%Q zO~GD#*xA+n;nV*9G)ozCqJgV(x2GiDuN%NOMmH%~qqMi|xG3v$t$|eZ2YO_IT4?{p2aT_zdPx)N7&8 zfRd7x-(2iR2?)|z6pR=$VlB?%a5(l8r@@k)J%czDFYj^tplZF2_8?ES!T$^(NW?K6 s&gJdh$Nl}y;qd2)CG}8_>$Tz51dBEi(Wrvqm_hwPz?_TAl|4WF4?wNCO#lD@ literal 0 HcmV?d00001 diff --git a/test/data/watershed-11chars.js b/test/data/watershed-11chars.js new file mode 100644 index 0000000..abd231e --- /dev/null +++ b/test/data/watershed-11chars.js @@ -0,0 +1,165 @@ +module.exports = [ { DWM_NAME: 'BUZZARDS BAY', + DWM_CODE: '95', + DRAINAGE910: 'coastal', + SHAPE_AREA: 1100426424.93, + SHAPE_LEN: 680071.913919 }, + { DWM_NAME: 'CAPE COD', + DWM_CODE: '96', + DRAINAGE910: 'coastal', + SHAPE_AREA: 913978642.943, + SHAPE_LEN: 802211.67478 }, + { DWM_NAME: 'MERRIMACK', + DWM_CODE: '84', + DRAINAGE910: 'river and coastal', + SHAPE_AREA: 728159472.653, + SHAPE_LEN: 330783.925198 }, + { DWM_NAME: 'DEERFIELD', + DWM_CODE: '33', + DRAINAGE910: 'river', + SHAPE_AREA: 897643472.911, + SHAPE_LEN: 162416.647504 }, + { DWM_NAME: 'CONNECTICUT', + DWM_CODE: '34', + DRAINAGE910: 'river', + SHAPE_AREA: 1733946880.89, + SHAPE_LEN: 418239.123507 }, + { DWM_NAME: 'NASHUA', + DWM_CODE: '81', + DRAINAGE910: 'river', + SHAPE_AREA: 1154375570.89, + SHAPE_LEN: 220748.885051 }, + { DWM_NAME: 'SHAWSHEEN', + DWM_CODE: '83', + DRAINAGE910: 'river', + SHAPE_AREA: 202238731.243, + SHAPE_LEN: 124382.780229 }, + { DWM_NAME: 'IPSWICH', + DWM_CODE: '92', + DRAINAGE910: 'river and coastal', + SHAPE_AREA: 402321528.158, + SHAPE_LEN: 194368.060063 }, + { DWM_NAME: 'CONCORD (SuAsCo)', + DWM_CODE: '82', + DRAINAGE910: 'river', + SHAPE_AREA: 1034986406.42, + SHAPE_LEN: 207648.657121 }, + { DWM_NAME: 'WESTFIELD', + DWM_CODE: '32', + DRAINAGE910: 'river', + SHAPE_AREA: 1337870479.81, + SHAPE_LEN: 309269.019544 }, + { DWM_NAME: 'HOUSATONIC', + DWM_CODE: '21', + DRAINAGE910: 'river', + SHAPE_AREA: 1295634364.97, + SHAPE_LEN: 254658.861254 }, + { DWM_NAME: 'CHARLES', + DWM_CODE: '72', + DRAINAGE910: 'river and coastal', + SHAPE_AREA: 804337278.228, + SHAPE_LEN: 249234.107364 }, + { DWM_NAME: 'BLACKSTONE', + DWM_CODE: '51', + DRAINAGE910: 'river', + SHAPE_AREA: 868716081.064, + SHAPE_LEN: 247406.544661 }, + { DWM_NAME: 'FARMINGTON', + DWM_CODE: '31', + DRAINAGE910: 'river', + SHAPE_AREA: 403208864.827, + SHAPE_LEN: 174077.861519 }, + { DWM_NAME: 'FRENCH', + DWM_CODE: '42', + DRAINAGE910: 'river', + SHAPE_AREA: 245240661.27, + SHAPE_LEN: 95630.237872 }, + { DWM_NAME: 'SOUTH COASTAL', + DWM_CODE: '94', + DRAINAGE910: 'coastal', + SHAPE_AREA: 623318341.503, + SHAPE_LEN: 348638.121478 }, + { DWM_NAME: 'QUINEBAUG', + DWM_CODE: '41', + DRAINAGE910: 'river', + SHAPE_AREA: 398412533.969, + SHAPE_LEN: 155488.644205 }, + { DWM_NAME: 'TAUNTON', + DWM_CODE: '62', + DRAINAGE910: 'river', + SHAPE_AREA: 1371098117.82, + SHAPE_LEN: 339730.645372 }, + { DWM_NAME: 'TEN MILE', + DWM_CODE: '52', + DRAINAGE910: 'river', + SHAPE_AREA: 125924435.756, + SHAPE_LEN: 84146.6302256 }, + { DWM_NAME: 'NARRAGANSETT BAY (SHORE)', + DWM_CODE: '53', + DRAINAGE910: 'coastal', + SHAPE_AREA: 150006557.671, + SHAPE_LEN: 63312.1134996 }, + { DWM_NAME: 'ISLANDS', + DWM_CODE: '97', + DRAINAGE910: 'coastal', + SHAPE_AREA: 412340033.574, + SHAPE_LEN: 443601.354619 }, + { DWM_NAME: 'CHICOPEE', + DWM_CODE: '36', + DRAINAGE910: 'river', + SHAPE_AREA: 1868726700.74, + SHAPE_LEN: 336390.398831 }, + { DWM_NAME: 'MILLERS', + DWM_CODE: '35', + DRAINAGE910: 'river', + SHAPE_AREA: 805953097.126, + SHAPE_LEN: 208548.57714 }, + { DWM_NAME: 'BOSTON HARBOR: Mystic', + DWM_CODE: '71', + DRAINAGE910: 'river and coastal', + SHAPE_AREA: 169490600.488, + SHAPE_LEN: 104702.983458 }, + { DWM_NAME: 'BOSTON HARBOR: Weymouth & Weir', + DWM_CODE: '74', + DRAINAGE910: 'river and coastal', + SHAPE_AREA: 230274616.059, + SHAPE_LEN: 205749.368678 }, + { DWM_NAME: 'BOSTON HARBOR: Neponset', + DWM_CODE: '73', + DRAINAGE910: 'river and coastal', + SHAPE_AREA: 302273378.57, + SHAPE_LEN: 164291.649994 }, + { DWM_NAME: 'NORTH COASTAL', + DWM_CODE: '93', + DRAINAGE910: 'coastal', + SHAPE_AREA: 444012782.854, + SHAPE_LEN: 481586.96927 }, + { DWM_NAME: 'PARKER', + DWM_CODE: '91', + DRAINAGE910: 'river and coastal', + SHAPE_AREA: 210671196.057, + SHAPE_LEN: 223674.642474 }, + { DWM_NAME: 'BOSTON HARBOR (Proper)', + DWM_CODE: '70', + DRAINAGE910: 'coastal', + SHAPE_AREA: 58322330.1896, + SHAPE_LEN: 192742.092623 }, + { DWM_NAME: 'MOUNT HOPE BAY (SHORE)', + DWM_CODE: '61', + DRAINAGE910: 'coastal', + SHAPE_AREA: 140721349.355, + SHAPE_LEN: 130978.457315 }, + { DWM_NAME: 'HUDSON: Bashbish', + DWM_CODE: '13', + DRAINAGE910: 'river', + SHAPE_AREA: 39802182.4217, + SHAPE_LEN: 41431.5176318 }, + { DWM_NAME: 'HUDSON: Hoosic', + DWM_CODE: '11', + DRAINAGE910: 'river', + SHAPE_AREA: 430987849.282, + SHAPE_LEN: 108535.221012 }, + { DWM_NAME: 'HUDSON: Kinderhook', + DWM_CODE: '12', + DRAINAGE910: 'river', + SHAPE_AREA: 56596528.9263, + SHAPE_LEN: 55533.0776528 } ]; \ No newline at end of file From b8449125dc2a35354eec8a9422be1697f84bc1a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0tefan=20Baebler?= Date: Thu, 9 Apr 2015 00:02:29 +0200 Subject: [PATCH 3/4] unit test covering special characters in field names --- test/data/watershed-specialCharacters.dbf | Bin 0 -> 3626 bytes test/data/watershed-specialCharacters.js | 165 ++++++++++++++++++++++ test/specialCharacters.test.js | 23 +++ 3 files changed, 188 insertions(+) create mode 100644 test/data/watershed-specialCharacters.dbf create mode 100644 test/data/watershed-specialCharacters.js create mode 100644 test/specialCharacters.test.js diff --git a/test/data/watershed-specialCharacters.dbf b/test/data/watershed-specialCharacters.dbf new file mode 100644 index 0000000000000000000000000000000000000000..9664da1aac49a169c9091d6d981b1c3552ff1326 GIT binary patch literal 3626 zcmai1OK&4Z5Vph#5GVG;sR0QEAzJ;Ye&sfHCb1QJCd}Bf*vl&HT1o`OOPod8pT|Gr z>v_eQOeQ{L`Z3dASJhWvRlfM=%QxRoCX@drH~8E8Z6}kn?dn~YtWxaE;_i>VSK|IG zKkN3t>g})7a?OAE^%NM@_iuXp8V}2zx{W2&#v!2up>mr zm6s-vQ`R&%nrq>_4$8>jYn-z*EzTF|a>O95pW?K$=IMEGmNamJl<`o{B)5vBwd9PT z#&3C+rL*;7wpsVS@20wEPq$GIBzIZ|vfc|{1tT!x(ksM9X(L>XlO^SRlk`4)I`y)h z*M4hf%@aH@q@6LY!%Zv=10dq8wS-b4f241qm?v~4OP%%d_Xi?4yyqbOA5iuM??#(OK3X?ZvP(0Mb9&!j5oMvqk zpYx8uy##!4rrDk+YSPVxAfU*|9`Qm>OrDY_hT70Mw5AO>*s|K<-E| zE*={52Ng+!76v?QaG-3f#qwpgF=k%FQ6?xNZJe?3&{*e4f@%nns+WjmNl_#hNmi!o zbyd4h%6VR-$NFPB9UY7hoFt%Q&}-`|5_{BbL<|KLdK4;XVvBN_WM~gYe!;bygE%NQ z$VnkI3~KU+IMhK%4OM7)J)19P`BggRH{JCIFVr88L{-+I^{8;F_*fl;4&Lh~PZ67C zT8!GGyMAX72fAZ|(IP;coA^MlJqTx#g}gUU_w;*_mEA0BSsw|=UhuE*=1 zzM&Zi6k@Fbm1`cLRFD%YAgSt-WLOVpyZik2{?o$^`-yFLxAzs0>+*E4&M7oE${`XK zcGWnr2eAq$b+*mZ9!|FVxW7B>9*!+?9p46VP;8V!E8^g%Mt5K$x;cqL5kjNaSzf%V zI~~pod?OOjHm?wq+8*MuhFj=EBYB2hCB Date: Fri, 15 May 2015 08:22:58 -0400 Subject: [PATCH 4/4] reorginize tests --- .travis.yml | 3 --- Gruntfile.js | 21 --------------------- component.json | 13 ------------- package.json | 10 ++++------ test/11characters.test.js | 23 ----------------------- test/basic.test.js | 29 +++++++++++++++++++++++++---- test/specialCharacters.test.js | 23 ----------------------- 7 files changed, 29 insertions(+), 93 deletions(-) delete mode 100644 Gruntfile.js delete mode 100644 component.json delete mode 100644 test/11characters.test.js delete mode 100644 test/specialCharacters.test.js diff --git a/.travis.yml b/.travis.yml index 7720259..ab98ab0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,3 @@ node_js: - "0.11" - "0.10" - "iojs" -before_install: npm install -g grunt-cli -install: npm install -#before_script: grunt test diff --git a/Gruntfile.js b/Gruntfile.js deleted file mode 100644 index 1100c6d..0000000 --- a/Gruntfile.js +++ /dev/null @@ -1,21 +0,0 @@ -module.exports = function(grunt) { - grunt.initConfig({ - pkg: grunt.file.readJSON('package.json'), - jshint: { - options: { - jshintrc: "./.jshintrc" - }, - all: ['./index.js'] - }, mochaTest: { - test: { - options: { - reporter: 'nyan' - }, - src: ['test/*.js'] - } - } - }); - grunt.loadNpmTasks('grunt-contrib-jshint'); - grunt.loadNpmTasks('grunt-mocha-test'); - grunt.registerTask('default', ['jshint','mochaTest']); -}; diff --git a/component.json b/component.json deleted file mode 100644 index d1a6563..0000000 --- a/component.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "name": "parsedbf", - "repo": "calvinmetcalf/parseDBF", - "description": "The DBF parsing component of shapefile-js", - "version": "0.0.1", - "keywords": [], - "development": {}, - "license": "MIT", - "main": "index.js", - "scripts": [ - "index.js" - ] -} diff --git a/package.json b/package.json index 42992b1..a3939cb 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "The DBF parsing component of shapefile-js", "main": "index.js", "scripts": { - "test": "grunt" + "test": "jshint index.js && mocha -R spec test/*.test.js" }, "repository": { "type": "git", @@ -20,10 +20,8 @@ "url": "https://github.com/calvinmetcalf/parseDBF/issues" }, "devDependencies": { - "grunt-mocha-test": "~0.7.0", - "grunt": "~0.4.1", - "grunt-contrib-jshint": "~0.6.4", - "mocha": "~1.13.0", - "chai": "~1.8.0" + "chai": "^2.3.0", + "jshint": "^2.7.0", + "mocha": "^2.2.5" } } diff --git a/test/11characters.test.js b/test/11characters.test.js deleted file mode 100644 index 22917fd..0000000 --- a/test/11characters.test.js +++ /dev/null @@ -1,23 +0,0 @@ -var fs = require('fs'); -var out = require('./data/watershed-11chars'); -var dbf = require('../'); -require('chai').should(); -function toArrayBuffer(buffer) { - var ab = new ArrayBuffer(buffer.length); - var view = new Uint8Array(ab); - for (var i = 0; i < buffer.length; ++i) { - view[i] = buffer[i]; - } - return ab; -} -describe('basic',function(){ - it('should work',function(done){ - fs.readFile('./test/data/watershed-11chars.dbf',function(err,data){ - if(err){ - return done(err); - } - dbf(toArrayBuffer(data)).should.deep.equal(out); - done(); - }); - }); -}); diff --git a/test/basic.test.js b/test/basic.test.js index 514e5cc..3338dac 100644 --- a/test/basic.test.js +++ b/test/basic.test.js @@ -1,5 +1,8 @@ var fs = require('fs'); -var out = require('./data/watershed'); +var basic = require('./data/watershed'); +var char11 = require('./data/watershed-11chars'); +var specialChar = require('./data/watershed-specialCharacters'); + var dbf = require('../'); require('chai').should(); function toArrayBuffer(buffer) { @@ -10,14 +13,32 @@ function toArrayBuffer(buffer) { } return ab; } -describe('basic',function(){ +describe('dbf',function(){ it('should work',function(done){ fs.readFile('./test/data/watershed.dbf',function(err,data){ if(err){ return done(err); } - dbf(toArrayBuffer(data)).should.deep.equal(out); + dbf(toArrayBuffer(data)).should.deep.equal(basic); done(); }); }); -}); \ No newline at end of file + it('should handle 11 charicter field names',function(done){ + fs.readFile('./test/data/watershed-11chars.dbf',function(err,data){ + if(err){ + return done(err); + } + dbf(toArrayBuffer(data)).should.deep.equal(char11); + done(); + }); + }); + it('should handle spetial charicters',function(done){ + fs.readFile('./test/data/watershed-specialCharacters.dbf',function(err,data){ + if(err){ + return done(err); + } + dbf(toArrayBuffer(data)).should.deep.equal(specialChar); + done(); + }); + }); +}); diff --git a/test/specialCharacters.test.js b/test/specialCharacters.test.js deleted file mode 100644 index 9f5c405..0000000 --- a/test/specialCharacters.test.js +++ /dev/null @@ -1,23 +0,0 @@ -var fs = require('fs'); -var out = require('./data/watershed-specialCharacters'); -var dbf = require('../'); -require('chai').should(); -function toArrayBuffer(buffer) { - var ab = new ArrayBuffer(buffer.length); - var view = new Uint8Array(ab); - for (var i = 0; i < buffer.length; ++i) { - view[i] = buffer[i]; - } - return ab; -} -describe('basic',function(){ - it('should work',function(done){ - fs.readFile('./test/data/watershed-specialCharacters.dbf',function(err,data){ - if(err){ - return done(err); - } - dbf(toArrayBuffer(data)).should.deep.equal(out); - done(); - }); - }); -});