diff --git a/.jshintrc b/.jshintrc index 159fb604..aa3f1574 100644 --- a/.jshintrc +++ b/.jshintrc @@ -2,7 +2,7 @@ "node": true, "curly": true, "eqeqeq": true, - "esversion": 6, + "esversion": 8, "freeze": true, "immed": true, "indent": 2, diff --git a/Dockerfile b/Dockerfile index fce05947..4ceb7ce4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # base image -FROM pelias/libpostal_baseimage +FROM pelias/baseimage # dependencies RUN apt-get update && \ diff --git a/api/extract.js b/api/extract.js index 4e9f187b..add15f0f 100644 --- a/api/extract.js +++ b/api/extract.js @@ -1,6 +1,7 @@ const Database = require('better-sqlite3'); const query = { extract: require('../query/extract') }; const analyze = require('../lib/analyze'); +const asyncForEach = require('../lib/asyncForEach'); // export setup method function setup( addressDbPath, streetDbPath ){ @@ -12,7 +13,7 @@ function setup( addressDbPath, streetDbPath ){ db.exec(`ATTACH DATABASE '${streetDbPath}' as 'street'`); // query method - var q = function( coord, names, cb ){ + var q = async function( coord, names, cb ){ var point = { lat: parseFloat( coord.lat ), @@ -20,8 +21,8 @@ function setup( addressDbPath, streetDbPath ){ }; var normalized = []; - names.forEach( function( name ){ - normalized = normalized.concat( analyze.street( name ) ); + await asyncForEach(names, async (name) => { + normalized = normalized.concat( await analyze.street( name ) ); }); // error checking diff --git a/api/search.js b/api/search.js index dfbfb829..02f02b3a 100644 --- a/api/search.js +++ b/api/search.js @@ -19,7 +19,7 @@ function setup( addressDbPath, streetDbPath ){ db.exec('PRAGMA street.mmap_size=268435456;'); // query method - var q = function( coord, number, street, cb ){ + var q = async function( coord, number, street, cb ){ var point = { lat: parseFloat( coord.lat ), @@ -31,7 +31,7 @@ function setup( addressDbPath, streetDbPath ){ var normalized = { number: analyze.housenumber( number ), - street: analyze.street( street ) + street: await analyze.street( street ) }; // error checking diff --git a/cmd/server.js b/cmd/server.js index 81e0d5f7..d2d7644b 100644 --- a/cmd/server.js +++ b/cmd/server.js @@ -220,10 +220,10 @@ app.use('/demo', express.static('demo')); // app.use('/builds', express.static('/data/builds')); // app.use('/builds', directory('/data/builds', { hidden: false, icons: false, view: 'details' })); -app.listen( PORT, HOST, function() { +app.listen( PORT, HOST, async function() { // force loading of libpostal - analyze.street( 'test street' ); + await analyze.street( 'test street' ); console.log(util.format( 'server listening on %s:%s', HOST || '0.0.0.0', PORT )); }); diff --git a/lib/analyze.js b/lib/analyze.js index cc13cf99..63bb21ce 100644 --- a/lib/analyze.js +++ b/lib/analyze.js @@ -1,41 +1,21 @@ +const postal = require('../libpostal/client'); + // constants for controlling how we parse ranges, eg: 'α-β' // some ranges such as '1-7' are ambiguous; it could mean 'apt 7, no 1'; or // it could mean 'apt 1, no 7'; or could even be a valid range 'one to seven'. // note: these values provide a means of setting some sane defaults for which // ranges we try to parse and which ones we leave. -var MIN_RANGE = 1; // the miniumum amount β is higher than α -var MAX_RANGE = 6; // the maximum amount β is higher than α -var MIN_RANGE_HOUSENUMBER = 10; // the minimum acceptible value for both α and β - -/* - * Return the appropriate version of node-postal - */ - -var _nodepostal_module; -function get_libpostal() { - // lazy load this dependency; since it's large (~2GB RAM) and may be - // accidentally required by a process which doesn't use it. - if (!_nodepostal_module) { - // load the mock library if MOCK_LIBPOSTAL env var is set - if (process.env.MOCK_LIBPOSTAL) { - _nodepostal_module = require('../test/lib/mock_libpostal'); - // otherwise load the real thing - } else { - _nodepostal_module = require('node-postal'); - } - } - - return _nodepostal_module; -} +const MIN_RANGE = 1; // the miniumum amount β is higher than α +const MAX_RANGE = 6; // the maximum amount β is higher than α +const MIN_RANGE_HOUSENUMBER = 10; // the minimum acceptible value for both α and β /** analyze input streetname string and return a list of expansions. **/ -function street( streetName ){ - const postal = get_libpostal(); +async function street( streetName ){ // use libpostal to expand the address - var expansions = postal.expand.expand_address( streetName ); + let expansions = await postal.expand.expand_address( streetName ); // remove ordinals expansions = expansions.map(function( item ){ diff --git a/lib/asyncForEach.js b/lib/asyncForEach.js new file mode 100644 index 00000000..69c6cec7 --- /dev/null +++ b/lib/asyncForEach.js @@ -0,0 +1,8 @@ +// async friendly version of Array.forEach +async function asyncForEach(array, callback) { + for (let index = 0; index < array.length; index++) { + await callback(array[index], index, array); + } +} + +module.exports = asyncForEach; diff --git a/libpostal/LibpostalServiceConfig.js b/libpostal/LibpostalServiceConfig.js new file mode 100644 index 00000000..489857e2 --- /dev/null +++ b/libpostal/LibpostalServiceConfig.js @@ -0,0 +1,17 @@ +const ServiceConfiguration = require('pelias-microservice-wrapper').ServiceConfiguration; + +class LibpostalServiceConfig extends ServiceConfiguration { + constructor(config) { + super('libpostal', config); + } + getUrl(params) { + return this.baseUrl + params.endpoint; + } + getParameters(params) { + return { + address: params.address + }; + } +} + +module.exports = LibpostalServiceConfig; diff --git a/libpostal/client.js b/libpostal/client.js new file mode 100644 index 00000000..12a42d64 --- /dev/null +++ b/libpostal/client.js @@ -0,0 +1,21 @@ +/* + * Return the appropriate version of node-postal + */ + +const config = require('pelias-config').generate(); +const serviceIsConfigured = config.get('services.libpostal') || config.get('api.services.libpostal'); + +// load the mock library if MOCK_LIBPOSTAL env var is set +if (process.env.MOCK_LIBPOSTAL) { + module.exports = require('./mock'); +} + +// else use the HTTP webservice when configured +else if (serviceIsConfigured) { + module.exports = require('./service'); +} + +// otherwise use the npm module +else { + module.exports = require('./module'); +} diff --git a/libpostal/fixture/libpostal_responses.json b/libpostal/fixture/libpostal_responses.json new file mode 100644 index 00000000..65645dd6 --- /dev/null +++ b/libpostal/fixture/libpostal_responses.json @@ -0,0 +1,1580 @@ +{ + "glasgow street": [ + "glasgow street" + ], + "grolmanstrasse": [ + "grolmanstraße", + "grolman straße" + ], + "west 26th street": [ + "west 26th street" + ], + "rigaer strasse": [ + "rigaer strasse" + ], + "markgrafenstrasse": [ + "markgrafenstraße", + "markgrafen straße" + ], + "potsdamer platz": [ + "potsdamer platz" + ], + "nevern square": [ + "nevern square" + ], + "cemetery road": [ + "cemetery road" + ], + "grolmanstraße": [ + "grolmanstraße", + "grolman straße" + ], + "southwest 26th street": [ + "southwest 26th street" + ], + "northwest 26th street": [ + "northwest 26th street" + ], + "cr 34": [ + "creek 34", + "county road 34", + "county route 34", + "crescent 34" + ], + "west 26th street place": [ + "west 26th street place" + ], + "us 20": [ + "us 20" + ], + "west 26th street road": [ + "west 26th street road" + ], + "west 26 street": [ + "west 26 street" + ], + "rigaer straße": [ + "rigaer strasse" + ], + "markgrafenstraße": [ + "markgrafenstraße", + "markgrafen straße" + ], + "potsdamer straße": [ + "potsdamer strasse" + ], + "l 69": [ + "left 69", + "lane 69", + "l 69", + "50 69", + "level 69", + "links 69", + "lang 69", + "lange 69" + ], + "alte potsdamer straße": [ + "alte potsdamer strasse" + ], + "b 1": [ + "b 1", + "bei 1" + ], + "s+u potsdamer platz": [ + "s+u potsdamer platz" + ], + "lützowstraße/potsdamer straße": [ + "luetzowstraße potsdamer strasse", + "luetzow straße potsdamer strasse", + "lutzowstraße potsdamer strasse", + "lutzow straße potsdamer strasse" + ], + "s potsdamer platz/voßstraße": [ + "sud potsdamer platz vossstraße", + "s potsdamer platz vossstraße", + "see potsdamer platz vossstraße", + "sud potsdamer platz voss straße", + "s potsdamer platz voss straße", + "see potsdamer platz voss straße" + ], + "potsdamer platz arkaden": [ + "potsdamer platz arkaden" + ], + "l 204": [ + "left 204", + "lane 204", + "l 204", + "50 204", + "level 204", + "links 204", + "lang 204", + "lange 204" + ], + "l 40": [ + "left 40", + "lane 40", + "l 40", + "50 40", + "level 40", + "links 40", + "lang 40", + "lange 40" + ], + "kleine potsdamer straße": [ + "kleine potsdamer strasse" + ], + "l 79": [ + "left 79", + "lane 79", + "l 79", + "50 79", + "level 79", + "links 79", + "lang 79", + "lange 79" + ], + "goethestraße/potsdamer straße": [ + "goethestraße potsdamer strasse", + "goethe straße potsdamer strasse" + ], + "b 273": [ + "b 273", + "bei 273" + ], + "k 6960": [ + "k 6960", + "kalea 6960", + "kamp 6960", + "katu 6960" + ], + "l 77": [ + "left 77", + "lane 77", + "l 77", + "50 77", + "level 77", + "links 77", + "lang 77", + "lange 77" + ], + "l 78": [ + "left 78", + "lane 78", + "l 78", + "50 78", + "level 78", + "links 78", + "lang 78", + "lange 78" + ], + "k 6909": [ + "k 6909", + "kalea 6909", + "kamp 6909", + "katu 6909" + ], + "l 90": [ + "left 90", + "lane 90", + "l 90", + "50 90", + "level 90", + "links 90", + "lang 90", + "lange 90" + ], + "l 86": [ + "left 86", + "lane 86", + "l 86", + "50 86", + "level 86", + "links 86", + "lang 86", + "lange 86" + ], + "l 92": [ + "left 92", + "lane 92", + "l 92", + "50 92", + "level 92", + "links 92", + "lang 92", + "lange 92" + ], + "b 102": [ + "b 102", + "bei 102" + ], + "k 7220": [ + "k 7220", + "kalea 7220", + "kamp 7220", + "katu 7220" + ], + "k 7221": [ + "k 7221", + "kalea 7221", + "kamp 7221", + "katu 7221" + ], + "k 7": [ + "k 7", + "kalea 7", + "kamp 7", + "katu 7" + ], + "willow avenue": [ + "willow avenue" + ], + "cr 634": [ + "creek 634", + "county road 634", + "county route 634", + "crescent 634" + ], + "brookwillow avenue": [ + "brookwillow avenue" + ], + "willow ave": [ + "willow avenue" + ], + "1016-18 willow avenue": [ + "1016-18 willow avenue" + ], + "1030-1032 willow avenue": [ + "1030-1032 willow avenue" + ], + "1109-1121 willow avenue": [ + "1109-1121 willow avenue" + ], + "112-14 willow avenue": [ + "112-14 willow avenue" + ], + "116-118 willow avenue": [ + "116-118 willow avenue" + ], + "1200-22 willow avenue": [ + "1200-22 willow avenue" + ], + "1203-19 willow avenue": [ + "1203-19 willow avenue" + ], + "124-128 willow avenue": [ + "124-128 willow avenue" + ], + "1300-14 willow avenue": [ + "1300-14 willow avenue" + ], + "1316-1330 willow avenue": [ + "1316-1330 willow avenue" + ], + "1317-1319 willow avenue": [ + "1317-1319 willow avenue" + ], + "1401-1407 willow avenue": [ + "1401-1407 willow avenue" + ], + "1404-12 willow avenue": [ + "1404-12 willow avenue" + ], + "1409-11 willow avenue": [ + "1409-11 willow avenue" + ], + "1413-1425 willow avenue": [ + "1413-1425 willow avenue" + ], + "1414-1418 willow avenue": [ + "1414-1418 willow avenue" + ], + "1420-1424 willow avenue": [ + "1420-1424 willow avenue" + ], + "1426-28 willow avenue": [ + "1426-28 willow avenue" + ], + "1427-1429 willow avenue": [ + "1427-1429 willow avenue" + ], + "147-51 willow avenue": [ + "147-51 willow avenue" + ], + "1501-1503 willow avenue": [ + "1501-1503 willow avenue" + ], + "1502-1506 willow avenue": [ + "1502-1506 willow avenue" + ], + "1508-1510 willow avenue": [ + "1508-1510 willow avenue" + ], + "1512-1522 willow avenue": [ + "1512-1522 willow avenue" + ], + "1524-1530 willow avenue": [ + "1524-1530 willow avenue" + ], + "1612-1614 willow avenue": [ + "1612-1614 willow avenue" + ], + "1622-28 willow avenue": [ + "1622-28 willow avenue" + ], + "167-169 willow avenue": [ + "167-169 willow avenue" + ], + "1700-02 willow avenue": [ + "1700-02 willow avenue" + ], + "1714-16 willow avenue": [ + "1714-16 willow avenue" + ], + "175-177 willow avenue": [ + "175-177 willow avenue" + ], + "1801-13 willow avenue": [ + "1801-13 willow avenue" + ], + "181-183 willow avenue": [ + "181-183 willow avenue" + ], + "182-184 willow avenue": [ + "182-184 willow avenue" + ], + "18oo willow avenue": [ + "18oo willow avenue" + ], + "19 willow avenue": [ + "19 willow avenue" + ], + "200-202 willow avenue": [ + "200-202 willow avenue" + ], + "201-215 willow avenue": [ + "201-215 willow avenue" + ], + "20-22 willow avenue": [ + "20-22 willow avenue" + ], + "203-205 willow avenue": [ + "203-205 willow avenue" + ], + "208-212 willow avenue": [ + "208-212 willow avenue" + ], + "214-216 willow avenue": [ + "214-216 willow avenue" + ], + "223-25 willow avenue": [ + "223-25 willow avenue" + ], + "224-226 willow avenue": [ + "224-226 willow avenue" + ], + "225a willow avenue": [ + "225a willow avenue" + ], + "27-29 willow avenue": [ + "27-29 willow avenue" + ], + "27 willow avenue": [ + "27 willow avenue" + ], + "28-30 willow avenue": [ + "28-30 willow avenue" + ], + "300-336 willow avenue": [ + "300-336 willow avenue" + ], + "308a willow avenue": [ + "308a willow avenue" + ], + "309-311 willow avenue": [ + "309-311 willow avenue" + ], + "311-313 willow avenue": [ + "311-313 willow avenue" + ], + "319a willow avenue": [ + "319a willow avenue" + ], + "321a willow avenue": [ + "321a willow avenue" + ], + "322a willow avenue": [ + "322a willow avenue" + ], + "32-34 willow avenue": [ + "32-34 willow avenue" + ], + "323a willow avenue": [ + "323a willow avenue" + ], + "324a willow avenue": [ + "324a willow avenue" + ], + "325-327 willow avenue": [ + "325-327 willow avenue" + ], + "33 willow avenue": [ + "33 willow avenue" + ], + "34 willow avenue": [ + "34 willow avenue" + ], + "400a willow avenue": [ + "400a willow avenue" + ], + "413a willow avenue": [ + "413a willow avenue" + ], + "424a willow avenue": [ + "424a willow avenue" + ], + "430a willow avenue": [ + "430a willow avenue" + ], + "453a willow avenue": [ + "453a willow avenue" + ], + "455a willow avenue": [ + "455a willow avenue" + ], + "458a willow avenue": [ + "458a willow avenue" + ], + "524-34 willow avenue": [ + "524-34 willow avenue" + ], + "529-533 willow avenue": [ + "529-533 willow avenue" + ], + "54-56 willow avenue": [ + "54-56 willow avenue" + ], + "5 willow avenue": [ + "5 willow avenue" + ], + "605-607 willow avenue": [ + "605-607 willow avenue" + ], + "619-621 willow avenue": [ + "619-621 willow avenue" + ], + "708-10 willow avenue": [ + "708-10 willow avenue" + ], + "708-710 willow avenue": [ + "708-710 willow avenue" + ], + "711-13 willow avenue": [ + "711-13 willow avenue" + ], + "712-14 willow avenue": [ + "712-14 willow avenue" + ], + "725-727 willow avenue": [ + "725-727 willow avenue" + ], + "730-732 willow avenue": [ + "730-732 willow avenue" + ], + "736-738 willow avenue": [ + "736-738 willow avenue" + ], + "77-83 willow avenue": [ + "77-83 willow avenue" + ], + "800-812 willow avenue": [ + "800-812 willow avenue" + ], + "832-834 willow avenue": [ + "832-834 willow avenue" + ], + "835-837 willow avenue": [ + "835-837 willow avenue" + ], + "89-91 willow avenue": [ + "89-91 willow avenue" + ], + "902-904 willow avenue": [ + "902-904 willow avenue" + ], + "913-915 willow avenue": [ + "913-915 willow avenue" + ], + "918-920 willow avenue": [ + "918-920 willow avenue" + ], + "end willow avenue": [ + "end willow avenue" + ], + "laurel place & willow avenue": [ + "laurel place & willow avenue" + ], + "maple & willow aves": [ + "maple & willow avenues" + ], + "west willow avenue": [ + "west willow avenue" + ], + "willow ave/257 13th": [ + "willow avenue 257 13th" + ], + "willow ave/260 eighth": [ + "willow avenue 260 8th" + ], + "willow ave/aka265 7th": [ + "willow avenue aka265 7th", + "willow avenue aka 265 7th" + ], + "willow avenue (a & b)": [ + "willow avenue a & b" + ], + "willow avenue extension": [ + "willow avenue extension" + ], + "willow avenue (lake)": [ + "willow avenue lake " + ], + "willow avenue & railroad": [ + "willow avenue & railroad" + ], + "willow avenue rear": [ + "willow avenue rear" + ], + "willow avenue (rear)": [ + "willow avenue rear " + ], + "willow avenue - rear": [ + "willow avenue rear" + ], + "nevern road": [ + "nevern road" + ], + "nevern place": [ + "nevern place" + ], + "lingshire rd": [ + "lingshire road" + ], + "smith river rd": [ + "smith river road" + ], + "butte creek rd": [ + "butte creek road" + ], + "fort logan rd": [ + "fort logan road" + ], + "hwy 360": [ + "highway 360" + ], + "state hwy 360": [ + "state hwy 360" + ], + "canyon rd": [ + "canyon road" + ], + "birch creek rd": [ + "birch creek road" + ], + "big sky ln": [ + "big sky lane", + "big sky line" + ], + "birky rd": [ + "birky road" + ], + "rostad rd": [ + "rostad road" + ], + "us hwy 89": [ + "us highway 89" + ], + "us hwy 12 e": [ + "us highway 12 east", + "us highway 12 e" + ], + "feddes rd": [ + "feddes road" + ], + "bonanza creek rd": [ + "bonanza creek road" + ], + "nat for dev rd 585": [ + "nat for dev road 585" + ], + "state hwy 294": [ + "state hwy 294" + ], + "cottonwood creek rd": [ + "cottonwood creek road" + ], + "findon ln": [ + "findon lane", + "findon line" + ], + "main st": [ + "main street", + "main saint" + ], + "grant ave": [ + "grant avenue" + ], + "b st": [ + "b st" + ], + "c st": [ + "center street", + "center saint", + "central street", + "central saint", + "c street", + "100 street", + "c saint", + "100 saint", + "centre street", + "centre saint" + ], + "merino ave": [ + "merino avenue", + "merino avenida" + ], + "2nd st": [ + "2nd street", + "2nd saint" + ], + "1st st": [ + "1st street", + "1st saint" + ], + "3rd ave": [ + "3rd avenue" + ], + "new dorsey rd": [ + "new dorsey road" + ], + "railroad ave": [ + "railroad avenue" + ], + "2nd ave": [ + "2nd avenue" + ], + "forest lake rd": [ + "forest lake road" + ], + "crazy m ranch rd": [ + "crazy m ranch road", + "crazy 1000 ranch road" + ], + "brekey rd": [ + "brekey road" + ], + "smith creek rd": [ + "smith creek road" + ], + "schendel rd": [ + "schendel road" + ], + "mike day dr": [ + "mike day drive", + "mike day doctor" + ], + "mountain view trl": [ + "mountain view trail" + ], + "sixteen mile rd": [ + "16 mile road" + ], + "co rd 119": [ + "county road 119" + ], + "sheep creek rd": [ + "sheep creek road" + ], + "newlan creek rd": [ + "newlan creek road" + ], + "stud horse rd": [ + "stud horse road" + ], + "bingham ln": [ + "bingham lane", + "bingham line" + ], + "jackson ln rd": [ + "jackson lane road", + "jackson line road" + ], + "monroe st": [ + "monroe street", + "monroe saint" + ], + "1st ave nw": [ + "1st avenue northwest", + "1st avenue nw" + ], + "n central ave": [ + "north central avenida", + "north central avenue", + "n central avenida", + "n central avenue", + "nosso central avenida", + "nosso central avenue", + "norte central avenida", + "norte central avenue" + ], + "1st ave ne": [ + "1st avenue ne", + "1st avenue nebraska", + "1st avenue northeast" + ], + "e baker st": [ + "east baker street", + "east baker saint", + "e baker street", + "e baker saint" + ], + "e grove st": [ + "east grove street", + "east grove saint", + "e grove street", + "e grove saint" + ], + "e woodson st": [ + "east woodson street", + "east woodson saint", + "e woodson street", + "e woodson saint" + ], + "2nd ave ne": [ + "2nd avenue ne", + "2nd avenue nebraska", + "2nd avenue northeast" + ], + "3rd ave ne": [ + "3rd avenue ne", + "3rd avenue nebraska", + "3rd avenue northeast" + ], + "e laramie st": [ + "east laramie street", + "east laramie saint", + "e laramie street", + "e laramie saint" + ], + "e wright st": [ + "east wright street", + "east wright saint", + "e wright street", + "e wright saint" + ], + "e washington st": [ + "east washington street", + "east washington saint", + "e washington street", + "e washington saint" + ], + "4th ave ne": [ + "4th avenue ne", + "4th avenue nebraska", + "4th avenue northeast" + ], + "5th ave ne": [ + "5th avenue ne", + "5th avenue nebraska", + "5th avenue northeast" + ], + "e hampton st": [ + "east hampton street", + "east hampton saint", + "e hampton street", + "e hampton saint" + ], + "6th ave ne": [ + "6th avenue ne", + "6th avenue nebraska", + "6th avenue northeast" + ], + "w main st": [ + "west main street", + "west main saint", + "w main street", + "w main saint" + ], + "e main st": [ + "east main street", + "east main saint", + "e main street", + "e main saint" + ], + "main st w": [ + "main street west", + "main street w", + "main saint west", + "main saint w" + ], + "2nd ave se": [ + "2nd avenue european company", + "2nd avenue southeast", + "2nd avenue suite", + "2nd avenue se" + ], + "3rd ave se": [ + "3rd avenue european company", + "3rd avenue southeast", + "3rd avenue suite", + "3rd avenue se" + ], + "e houston st": [ + "east houston street", + "east houston saint", + "e houston street", + "e houston saint" + ], + "4th ave se": [ + "4th avenue european company", + "4th avenue southeast", + "4th avenue suite", + "4th avenue se" + ], + "e lincoln st": [ + "east lincoln street", + "east lincoln saint", + "e lincoln street", + "e lincoln saint" + ], + "5th ave se": [ + "5th avenue european company", + "5th avenue southeast", + "5th avenue suite", + "5th avenue se" + ], + "1st ave se": [ + "1st avenue european company", + "1st avenue southeast", + "1st avenue suite", + "1st avenue se" + ], + "e chilton st": [ + "east chilton street", + "east chilton saint", + "e chilton street", + "e chilton saint" + ], + "e jefferson st": [ + "east jefferson street", + "east jefferson saint", + "e jefferson street", + "e jefferson saint" + ], + "10th ave sw": [ + "10th avenue southwest", + "10th avenue sw" + ], + "s central ave": [ + "south central avenida", + "south central avenue", + "san central avenida", + "san central avenue", + "s central avenida", + "s central avenue", + "sul central avenida", + "sul central avenue", + "sao central avenida", + "sao central avenue", + "senhor central avenida", + "senhor central avenue" + ], + "e alabama st": [ + "east alabama street", + "east alabama saint", + "e alabama street", + "e alabama saint" + ], + "e brown st": [ + "east brown street", + "east brown saint", + "e brown street", + "e brown saint" + ], + "5th ave sw": [ + "5th avenue southwest", + "5th avenue sw" + ], + "e crawford st": [ + "east crawford street", + "east crawford saint", + "e crawford street", + "e crawford saint" + ], + "sw maginnis st": [ + "southwest maginnis street", + "southwest maginnis saint", + "sw maginnis street", + "sw maginnis saint" + ], + "4th ave sw": [ + "4th avenue southwest", + "4th avenue sw" + ], + "3rd ave sw": [ + "3rd avenue southwest", + "3rd avenue sw" + ], + "e maginnis st": [ + "east maginnis street", + "east maginnis saint", + "e maginnis street", + "e maginnis saint" + ], + "folsom st": [ + "folsom street", + "folsom saint" + ], + "sw folsom st": [ + "southwest folsom street", + "southwest folsom saint", + "sw folsom street", + "sw folsom saint" + ], + "se folsom st": [ + "european company folsom street", + "european company folsom saint", + "southeast folsom street", + "southeast folsom saint", + "suite folsom street", + "suite folsom saint", + "se folsom street", + "se folsom saint" + ], + "w folsom st": [ + "west folsom street", + "west folsom saint", + "w folsom street", + "w folsom saint" + ], + "e folsom st": [ + "east folsom street", + "east folsom saint", + "e folsom street", + "e folsom saint" + ], + "e hancock st": [ + "east hancock street", + "east hancock saint", + "e hancock street", + "e hancock saint" + ], + "sw garfield st": [ + "southwest garfield street", + "southwest garfield saint", + "sw garfield street", + "sw garfield saint" + ], + "w garfield st": [ + "west garfield street", + "west garfield saint", + "w garfield street", + "w garfield saint" + ], + "e garfield st": [ + "east garfield street", + "east garfield saint", + "e garfield street", + "e garfield saint" + ], + "e south st": [ + "east south street", + "east south saint", + "e south street", + "e south saint" + ], + "6th ave sw": [ + "6th avenue southwest", + "6th avenue sw" + ], + "slaughter house rd": [ + "slaughter house road" + ], + "ramspeck ln": [ + "ramspeck lane", + "ramspeck line", + "ramspeck laan" + ], + "jackson ln": [ + "jackson lane", + "jackson line" + ], + "slaughter house ln": [ + "slaughter house lane", + "slaughter house line" + ], + "luppold rd": [ + "luppold road" + ], + "1st ave sw": [ + "1st avenue southwest", + "1st avenue sw" + ], + "sw 1st ave": [ + "southwest 1st avenue", + "sw 1st avenue" + ], + "sw south st": [ + "southwest south street", + "southwest south saint", + "sw south street", + "sw south saint" + ], + "2nd ave sw": [ + "2nd avenue southwest", + "2nd avenue sw" + ], + "w hampton st": [ + "west hampton street", + "west hampton saint", + "w hampton street", + "w hampton saint" + ], + "swmaginnis st": [ + "swmaginnis street", + "swmaginnis saint" + ], + "9th ave nw": [ + "9th avenue northwest", + "9th avenue nw" + ], + "jefferson st": [ + "jefferson street", + "jefferson saint" + ], + "castle mtn ranch rd": [ + "castle mountain ranch road" + ], + "fourmile creek rd": [ + "fourmile creek road" + ], + "nat for dev rd 211": [ + "nat for dev road 211" + ], + "castle mountain estate rd": [ + "castle mountain estate road" + ], + "castle mnt ests rd": [ + "castle mount estates road" + ], + "apple rd": [ + "apple road" + ], + "forest rd": [ + "forest road" + ], + "cold springs rd": [ + "cold springs road" + ], + "spring creek rd": [ + "spring creek road" + ], + "natl forest develop road 3423 rd": [ + "national forest develop road 3423 road" + ], + "ranch creek rd": [ + "ranch creek road" + ], + "ranch creek rd s": [ + "ranch creek road south", + "ranch creek road san", + "ranch creek road s" + ], + "w houston st": [ + "west houston street", + "west houston saint", + "w houston street", + "w houston saint" + ], + "cedar rd": [ + "cedar road" + ], + "birch rd": [ + "birch road" + ], + "shearer rd": [ + "shearer road" + ], + "luppold ln": [ + "luppold laan", + "luppold lane", + "luppold line" + ], + "earling ave": [ + "earling avenue" + ], + "ryan": [ + "ryan" + ], + "miller rd": [ + "miller road" + ], + "meadow creek rd": [ + "meadow creek road" + ], + "kiff rd": [ + "kiff road" + ], + "voldseth rd": [ + "voldseth road" + ], + "n badger": [ + "north badger", + "n badger" + ], + "wall st": [ + "wall street", + "wall saint" + ], + "little moose creek trl": [ + "little moose creek trail" + ], + "5th ave nw": [ + "5th avenue northwest", + "5th avenue nw" + ], + "sw hancock st": [ + "southwest hancock street", + "southwest hancock saint", + "sw hancock street", + "sw hancock saint" + ], + "sw crawford st": [ + "southwest crawford street", + "southwest crawford saint", + "sw crawford street", + "sw crawford saint" + ], + "8th ave sw": [ + "8th avenue southwest", + "8th avenue sw" + ], + "w alabama st": [ + "west alabama street", + "west alabama saint", + "w alabama street", + "w alabama saint" + ], + "forest rd south": [ + "forest road south" + ], + "us hwy 89 n": [ + "us highway 89 north", + "us highway 89 n" + ], + "west roadway": [ + "west roadway" + ], + "stagecoach trl": [ + "stagecoach trail" + ], + "mountain rd": [ + "mountain road" + ], + "2nd ave s": [ + "2nd avenue south", + "2nd avenue san", + "2nd avenue s" + ], + "bridger view cir": [ + "bridger view circle" + ], + "lucas rd": [ + "lucas road" + ], + "canyon ranch rd": [ + "canyon ranch road" + ], + "sunrise dr": [ + "sunrise drive", + "sunrise doctor" + ], + "jackrabbit ln": [ + "jackrabbit lane", + "jackrabbit line" + ], + "sagebrush trl": [ + "sagebrush trail" + ], + "arrowhead cir": [ + "arrowhead circle" + ], + "grassy mountain rd": [ + "grassy mountain road" + ], + "pine hill dr": [ + "pine hill drive", + "pine hill doctor" + ], + "cemetery rd": [ + "cemetery road" + ], + "w south st": [ + "west south street", + "west south saint", + "w south street", + "w south saint" + ], + "badger st": [ + "badger street", + "badger saint" + ], + "e monroe st": [ + "east monroe street", + "east monroe saint", + "e monroe street", + "e monroe saint" + ], + "studhorse rd": [ + "studhorse road" + ], + "folsom st w": [ + "folsom street west", + "folsom street w", + "folsom saint west", + "folsom saint w" + ], + "w chilton st": [ + "west chilton street", + "west chilton saint", + "w chilton street", + "w chilton saint" + ], + "grove st": [ + "grove street", + "grove saint" + ], + "lind ln": [ + "lind lane", + "lind line", + "lind laan" + ], + "baker st w": [ + "baker street west", + "baker street w", + "baker saint west", + "baker saint w" + ], + "south st e": [ + "south street east", + "south street e", + "south saint east", + "south saint e" + ], + "1st st s": [ + "1st street south", + "1st street san", + "1st street s", + "1st saint south", + "1st saint san", + "1st saint s" + ], + "lake rd": [ + "lake road" + ], + "mountainview ln": [ + "mountainview lane", + "mountainview line" + ], + "bachler ln": [ + "bachler lane", + "bachler line", + "bachler laan" + ], + "ryan st": [ + "ryan street", + "ryan saint" + ], + "allen gulch rd": [ + "allen gulch road" + ], + "maudlow rd": [ + "maudlow road" + ], + "s east rd": [ + "southeast road" + ], + "hitching post rd": [ + "hitching post road" + ], + "hidden trl": [ + "hidden trail" + ], + "3 rd": [ + "3 road" + ], + "goat mountain rd": [ + "goat mountain road" + ], + "sage brush trl": [ + "sage brush trail" + ], + "south st": [ + "south street", + "south saint" + ], + "zehntner ln": [ + "zehntner lane", + "zehntner line" + ], + "painted pony ln": [ + "painted pony lane", + "painted pony line" + ], + "n star rd": [ + "north star road", + "n star road" + ], + "grande rd": [ + "grande road" + ], + "goat mountain f": [ + "goat mountain f", + "goat mountain front" + ], + "panorama dr": [ + "panorama drive", + "panorama doctor" + ], + "airport rd": [ + "airport road" + ], + "2nd st s": [ + "2nd street south", + "2nd street san", + "2nd street s", + "2nd saint south", + "2nd saint san", + "2nd saint s" + ], + "berg rd": [ + "berg road" + ], + "castle mt rd": [ + "castle mt road", + "castle mount road", + "castle montana road" + ], + "castle mountain rd": [ + "castle mountain road" + ], + "brown st": [ + "brown street", + "brown saint" + ], + "crawford st w": [ + "crawford street west", + "crawford street w", + "crawford saint west", + "crawford saint w" + ], + "sw10th ave": [ + "sw10th avenue", + "southwest 10th avenue", + "sw 10th avenue" + ], + "castle town rd": [ + "castle town road" + ], + "nat for dev rd 581": [ + "nat for dev road 581" + ], + "lennep rd": [ + "lennep road" + ], + "elk ridge rd": [ + "elk ridge road" + ], + "71 ranch rd": [ + "71 ranch road" + ], + "crawford st": [ + "crawford street", + "crawford saint" + ], + "fox wood ln": [ + "fox wood lane", + "fox wood line" + ], + "foxwood ln": [ + "foxwood lane", + "foxwood line" + ], + "west 26th st": [ + "west 26th street", + "west 26th saint" + ], + "": [ + "" + ], + "lueppold road": [ + "lueppold road" + ], + "lueppold rd": [ + "lueppold road" + ], + "cow coulee rd": [ + "cow coulee road" + ], + "nfs rd 586": [ + "nfs road 586" + ], + "chris aren rd": [ + "chris aren road" + ], + "whitetail rd": [ + "whitetail road" + ], + "camas creek rd": [ + "camas creek road" + ], + "battle creek rd": [ + "battle creek road" + ], + "bozeman fork rd": [ + "bozeman fork road" + ], + "e larime st": [ + "east larime saint", + "east larime street", + "e larime saint", + "e larime street" + ], + "us hwy 12": [ + "us highway 12" + ], + "moose creek rd": [ + "moose creek road" + ], + "smokey mountain cir": [ + "smokey mountain circle" + ], + "1st ave s": [ + "1st avenue san", + "1st avenue s", + "1st avenue south", + "1 avenue san", + "1 avenue s", + "1 avenue south" + ], + "e smith crk": [ + "east smith creek", + "e smith creek" + ], + "lincoln st": [ + "lincoln saint", + "lincoln street" + ], + "grasshopper": [ + "grasshopper" + ], + "spencer ave": [ + "spencer avenue" + ], + "6th ave nw": [ + "6th avenue northwest", + "6th avenue nw", + "6 avenue northwest", + "6 avenue nw" + ], + "hampton st w": [ + "hampton saint west", + "hampton saint w", + "hampton street west", + "hampton street w" + ], + "10th ave nw": [ + "10th avenue northwest", + "10th avenue nw", + "10 avenue northwest", + "10 avenue nw" + ], + "black butte rd": [ + "black butte road" + ], + "1 rd": [ + "1 road" + ], + "nfs rd 211": [ + "nfs road 211" + ], + "4th ave nw": [ + "4th avenue northwest", + "4th avenue nw", + "4 avenue northwest", + "4 avenue nw" + ], + "staff rd": [ + "staff road" + ], + "2 rd": [ + "2 road" + ], + "mayn creek rd": [ + "mayn creek road" + ], + "spring creek trl": [ + "spring creek trail" + ], + "6 rd": [ + "6 road" + ], + "sw houston st": [ + "southwest houston saint", + "southwest houston street", + "sw houston saint", + "sw houston street" + ], + "7th ave nw": [ + "7th avenue northwest", + "7th avenue nw", + "7 avenue northwest", + "7 avenue nw" + ], + "gipsy ln": [ + "gipsy lane", + "gipsy line" + ], + "sunset ln": [ + "sunset lane", + "sunset line" + ], + "two creeks rd": [ + "2 creeks road" + ], + "indian creek ln": [ + "indian creek lane", + "indian creek line" + ], + "sixteen ln": [ + "16 lane", + "16 line", + "16 laan" + ], + "hall creek rd": [ + "hall creek road" + ], + "blue bird ln": [ + "blue bird lane", + "blue bird line" + ], + "newlan trl": [ + "newlan trail" + ], + "brewer ln": [ + "brewer lane", + "brewer line" + ], + "lower sixteen mile rd": [ + "lower 16 mile road" + ] +} \ No newline at end of file diff --git a/libpostal/mock.js b/libpostal/mock.js new file mode 100644 index 00000000..229ac94e --- /dev/null +++ b/libpostal/mock.js @@ -0,0 +1,45 @@ +const _ = require('lodash'); +const fs = require('fs'); +const path = require('path'); +const fixtureFile = path.join(__dirname, './fixture/libpostal_responses.json'); +let fixtures = require(fixtureFile); + +/* When `SEED_MOCK_LIBPOSTAL is set, this library will actually + * call through to the real libpostal and record the response. + * In this way the mock responses can be kept up to date as libpostal changes */ +const SEED_MOCK_LIBPOSTAL = _.has(process, 'env.SEED_MOCK_LIBPOSTAL'); + +const expand_address = async (address) => { + + // perform some basic normalization on the address string + const normalizedAddress = address.trim().toLowerCase(); + + // return a mocked response if one is available + const mockResponse = _.get(fixtures, normalizedAddress); + if (mockResponse) { + return Promise.resolve(mockResponse); + } + + // if no mock response is available but falling back to libpostal service + // is enabled, and return the real response + else if (SEED_MOCK_LIBPOSTAL) { + const service = require('./service'); + const resp = await service.expand.expand_address(normalizedAddress); + + // write the stored list of responses after _every_ new one is added. this is inefficient + // but it does not appear using `process.on('exit')` is reliable + fixtures[normalizedAddress] = resp; + fs.writeFileSync(fixtureFile, JSON.stringify(fixtures, null, 2)); + + return Promise.resolve(resp); + } + + // if there is no mock response and falling back to real libpostal is disabled, + // throw an error because a human has to run libpostal and find the correct response + else { + console.error(`mock libpostal has no response for ${normalizedAddress}`); + process.exit(1); + } +}; + +module.exports.expand = { expand_address }; diff --git a/libpostal/module.js b/libpostal/module.js new file mode 100644 index 00000000..be74dca7 --- /dev/null +++ b/libpostal/module.js @@ -0,0 +1,17 @@ +let postal; + +// lazy-load node-postal only as required +const lazy = () => { + if(!postal){ postal = require('node-postal'); } + return postal; +}; + +const expand_address = async (address) => { + // return empty array immediately for empty input + if (!address) { return Promise.resolve([]); } + + const resp = lazy().expand.expand_address(address); + return Promise.resolve(resp); +}; + +module.exports.expand = { expand_address }; diff --git a/libpostal/service.js b/libpostal/service.js new file mode 100644 index 00000000..5b39f00b --- /dev/null +++ b/libpostal/service.js @@ -0,0 +1,31 @@ +const util = require('util'); +const pmw = require('pelias-microservice-wrapper'); +const LibpostalServiceConfig = require('./LibpostalServiceConfig'); +const config = require('pelias-config').generate(); + +// use the 'services.libpostal' config entry if available, otherwise fall back to 'api.services.libpostal' +const serviceConfig = config.get('services.libpostal') || config.get('api.services.libpostal'); +if (!serviceConfig) { + throw new Error('Libpostal configuration not found in `services.libpostal` or `api.services.libpostal`'); +} + +// create a service +const service = pmw.service(new LibpostalServiceConfig(serviceConfig)); + +// create an object that looks like the interface to `node-postal` but uses a remote service +module.exports = { + expand: { + expand_address: async function (address) { + + // the libpostal service will not handle an empty address + // string, so return empty array immediately + if (!address) { return Promise.resolve([]); } + + const promise = util.promisify(service); + return promise({ + endpoint: 'expand', + address + }); + } + } +}; diff --git a/package.json b/package.json index 2d8574e6..7a85bf81 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "pbf2json": "^6.4.0", "pelias-config": "^4.0.0", "pelias-logger": "^1.2.1", + "pelias-microservice-wrapper": "^1.8.3", "quadtree": "^1.1.3", "require-dir": "^1.0.0", "serve-index": "^1.8.0", diff --git a/readme.md b/readme.md index 679a50ca..5f054cc9 100644 --- a/readme.md +++ b/readme.md @@ -263,7 +263,6 @@ see: [source](https://github.com/pelias/interpolation/blob/master/cmd/server.js) ## docker ### build docker image -This can take some time for the first build due to installing libpostal from source. ```bash docker build -t pelias/interpolation . ``` @@ -463,8 +462,6 @@ To use Interpolation service with the Pelias API, [configure the pelias config f ### install dependencies -*note:* [libpostal](https://github.com/openvenues/node-postal#troubleshooting) **must** be installed on your system before you continue! - The `Dockerfile` in this repo has complete instructions on how to install everything from scratch on Ubuntu. ### TIGER dependency on GDAL diff --git a/stream/address/lookup.js b/stream/address/lookup.js index 45400803..10d0fee1 100644 --- a/stream/address/lookup.js +++ b/stream/address/lookup.js @@ -17,7 +17,7 @@ if( hasFD3 ){ function streamFactory(db){ - return through.obj(function( batch, _, next ){ + return through.obj(async function( batch, _, next ){ // invalid batch if( !batch || !batch.length ){ @@ -30,7 +30,7 @@ function streamFactory(db){ // all street names in batch should be the same // perform libpostal normalization - var names = analyze.street( result.getStreet() ); + var names = await analyze.street( result.getStreet() ); // ensure at least one name was produced if( !names.length ){ diff --git a/stream/street/augment.js b/stream/street/augment.js index 84564222..31e7bdf2 100644 --- a/stream/street/augment.js +++ b/stream/street/augment.js @@ -1,5 +1,6 @@ const through = require('through2'); const analyze = require('../../lib/analyze'); +const asyncForEach = require('../../lib/asyncForEach'); // increase/decrease bbox bounds by this much in order to find houses which // might be slighly outside the bounds. @@ -15,12 +16,12 @@ const FUDGE_FACTOR = 0.005; **/ // the transform function is executed once per batch in the stream. -const transform = (street, _, next) => { +const transform = async (street, _, next) => { // normalize all names let names = []; - street.getNames().forEach(function (name) { - names = names.concat(analyze.street(name)); + await asyncForEach(street.getNames(), async (name) => { + names = names.concat(await analyze.street(name)); }); // if the source file contains no valid names for this polyline diff --git a/test/lib/analyze.js b/test/lib/analyze.js index 9ae15f6f..8590cbb0 100644 --- a/test/lib/analyze.js +++ b/test/lib/analyze.js @@ -4,18 +4,18 @@ var analyze = require('../../lib/analyze'); module.exports.analyze = {}; module.exports.analyze.street = function(test) { - test('street: synonym expansions', function(t) { - var perms = analyze.street('grolmanstraße'); + test('street: synonym expansions', async function(t) { + var perms = await analyze.street('grolmanstraße'); t.deepEqual(perms, ['grolmanstraße', 'grolman straße']); t.end(); }); - test('street: remove ordinals', function(t) { - var perms = analyze.street('West 26th st'); + test('street: remove ordinals', async function(t) { + var perms = await analyze.street('West 26th st'); t.deepEqual(perms, ['west 26 street', 'west 26 saint']); t.end(); }); - test('street: always returns array', function(t) { - var perms = analyze.street(''); + test('street: always returns array', async function(t) { + var perms = await analyze.street(''); t.deepEqual(perms, ['']); t.end(); }); diff --git a/test/lib/mock_libpostal.js b/test/lib/mock_libpostal.js deleted file mode 100644 index 9d064750..00000000 --- a/test/lib/mock_libpostal.js +++ /dev/null @@ -1,42 +0,0 @@ -const _ = require('lodash'); -const fs = require('fs'); - -// the real libpostal module, if needed will be loaded here -let real_libpostal; - -/* When `SEED_MOCK_LIBPOSTAL is set, this library will actually - * call through to the real libpostal and record the response. - * In this way the mock responses can be kept up to date as libpostal changes */ -const use_real_libpostal = process.env.SEED_MOCK_LIBPOSTAL !== undefined; - -// put all desired responses from libpostal here -let mock_responses = require('../../test/lib/mock_libpostal_responses'); - -module.exports.expand = { - expand_address: function(input_string) { - const clean_string = input_string.trim().toLowerCase(); - // return a mocked response if one is available - if (_.has(mock_responses, clean_string)) { - return mock_responses[clean_string]; - // if no mock response is available but falling back to real libpostal - // is enabled, lazy load real libpostal, and return the real response - } else if (use_real_libpostal) { - // lazy load libpostal only when needed - if (!real_libpostal) { real_libpostal = require('node-postal'); } - - const real_response = real_libpostal.expand.expand_address(clean_string); - mock_responses[clean_string] = real_response; - - // write the stored list of responses after _every_ new one is added. this is inefficient - // but it does not appear using `process.on('exit')` is reliable - fs.writeFileSync(__dirname +'/../../test/lib/mock_libpostal_responses.json', JSON.stringify(mock_responses, null, 2)); - - return real_response; - // if there is no mock response and falling back to real libpostal is disabled, - // throw an error because a human has to run libpostal and find the correct response - } else { - console.error(`mock libpostal has no response for ${clean_string}`); - process.exit(1); - } - } -};