Skip to content

Commit

Permalink
feat(libpostal): Use libpostal service
Browse files Browse the repository at this point in the history
BREAKING CHANGE

Use microservice-wrapper to avoid having to load libpostal locally.

Note: this now requires a new configuration section in `pelias.json`, a
top-level `services` key with the usual properties. Here's an example
full `pelias.json`:

```
{
  "api": {
    "textAnalyzer": "libpostal"
  },
  "services": {
    "libpostal": {
      "url": "http://libpostal-service-url:8080",
      "timeout": 4000
    }
  }
}
```

Fixes #106
  • Loading branch information
orangejulius committed Sep 17, 2019
1 parent 3edc5a9 commit d6c91fc
Show file tree
Hide file tree
Showing 5 changed files with 74 additions and 23 deletions.
24 changes: 2 additions & 22 deletions lib/analyze.js
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
const libpostal_service = require( './libpostal_wrapper' );
// constants for controlling how we parse ranges, eg: 'α-β'
// some ranges such as '1-7' are ambiguous; it could mean 'apt 7, no 1'; or
// it could mean 'apt 1, no 7'; or could even be a valid range 'one to seven'.
Expand All @@ -7,32 +8,11 @@ var MIN_RANGE = 1; // the miniumum amount β is higher than α
var MAX_RANGE = 6; // the maximum amount β is higher than α
var MIN_RANGE_HOUSENUMBER = 10; // the minimum acceptible value for both α and β

/*
* Return the appropriate version of node-postal
*/

var _nodepostal_module;
function get_libpostal() {
// lazy load this dependency; since it's large (~2GB RAM) and may be
// accidentally required by a process which doesn't use it.
if (!_nodepostal_module) {
// load the mock library if MOCK_LIBPOSTAL env var is set
if (process.env.MOCK_LIBPOSTAL) {
_nodepostal_module = require('../test/lib/mock_libpostal');
// otherwise load the real thing
} else {
_nodepostal_module = require('node-postal');
}
}

return _nodepostal_module;
}

/**
analyze input streetname string and return a list of expansions.
**/
function street( streetName ){
const postal = get_libpostal();
const postal = libpostal_service();

// use libpostal to expand the address
var expansions = postal.expand.expand_address( streetName );
Expand Down
22 changes: 22 additions & 0 deletions lib/libpostal_wrapper.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
const mock_libpostal = require('../test/lib/mock_libpostal');

// This module is a wrapper around the actual libpostal service library
// and the mock libpostal library
// it allows an environment variable to switch which library is used in application code

let libpostal_module;
function get_libpostal() {
// return the mock library if MOCK_LIBPOSTAL env var is set
if (process.env.MOCK_LIBPOSTAL) {
return mock_libpostal;
// otherwise return the actual service
} else {
// lazy load the libpostal module so that tests can skip configuring the service
if (!libpostal_module) {
libpostal_module = require( '../libpostal/service' );
}
return libpostal_module;
}
}

module.exports = get_libpostal;
48 changes: 48 additions & 0 deletions libpostal/service.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
// deasync is used to proved a sync-looking interface
// to the async call to the libpostal service
const deasync = require('deasync');
const microservice_wrapper = require('pelias-microservice-wrapper');
const pelias_config = require('pelias-config').generate();

if (!pelias_config.services || !pelias_config.services.libpostal || !pelias_config.services.libpostal.url) {
throw new Error('The URL to the libpostal service must be set in the services.libpostal.url property of pelias.json');
}

const LibpostalServiceConfig = class extends microservice_wrapper.ServiceConfiguration {
constructor(configBlob) {
super('libpostal', configBlob);
}
getUrl(params) {
return this.baseUrl + params.endpoint;
}
getParameters(params) {
return {
address: params.address
};
}
};

// create an instance of the libpostal service, with a synchronous interface
const libpostal_service = deasync(microservice_wrapper.service(
new LibpostalServiceConfig(pelias_config.services.libpostal)
));

// create an object that looks like the interface to `node-postal` but uses a remote service
module.exports = {
expand: {
expand_address: function(param) {
const params = {
endpoint: 'expand',
address: param
};

// the libpostal service will not handle an empty parameter
// so return empty array immediately
if (!param) {
return [];
}
const result = libpostal_service(params);
return result;
}
}
};
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
"async": "^3.1.0",
"cli-table3": "^0.5.0",
"csv-parse": "^4.0.0",
"deasync": "^0.1.15",
"express": "^4.14.0",
"fs-extra": "^8.0.0",
"morgan": "^1.9.0",
Expand Down
2 changes: 1 addition & 1 deletion test/lib/mock_libpostal.js
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ module.exports.expand = {
// is enabled, lazy load real libpostal, and return the real response
} else if (use_real_libpostal) {
// lazy load libpostal only when needed
if (!real_libpostal) { real_libpostal = require('node-postal'); }
if (!real_libpostal) { real_libpostal = require('../../libpostal/service'); }

const real_response = real_libpostal.expand.expand_address(clean_string);
mock_responses[clean_string] = real_response;
Expand Down

0 comments on commit d6c91fc

Please sign in to comment.