feat(multi-lang): Add index for all wof languages (when differents to…

… default language)
pelias · May 16, 2019 · a3f2849 · a3f2849
1 parent b831c23
commit a3f2849
Show file tree

Hide file tree

Showing 4 changed files with 40 additions and 5 deletions.
diff --git a/src/components/extractFields.js b/src/components/extractFields.js
@@ -1,6 +1,7 @@
 const through2 = require('through2');
 const _ = require('lodash');
 const util = require('util');
+const iso3166 = require('iso3166-1');
 
 // hierarchy in importance-descending order of population fields
 const population_hierarchy = [
@@ -26,6 +27,8 @@ const NAME_ALIAS_FIELDS = [
   'label:%s_x_preferred'
 ];
 
+const wofNamesRegex = /name:[a-z]{3}_x_preferred/;
+
 // this function is used to verify that a US county QS altname is available
 function isUsCounty(base_record, wof_country, qs_a2_alt) {
   return 'US' === wof_country &&
@@ -131,6 +134,22 @@ function getNameAliases(properties) {
   return concatArrayFields(properties, nameFields);
 }
 
+function getMultiLangNames(defaultName, properties) {
+  return Object.keys(properties)
+    .filter(key => wofNamesRegex.test(key)) // get only name:.* keys
+    .map(key => {
+      return {
+        key: key.substring(5, 8).toUpperCase(), // get the iso part of the key name:iso_x_preferred
+        value: properties[key]
+                .filter(name => !defaultName || defaultName.indexOf(name) < 0) // remove duplicate elements found in default name
+      };
+    }) //
+    .filter(({ key, value }) => value.length > 0 && iso3166.is3(key)) // filter correct iso 3 keys
+    .reduce((langs, { key, value }) =>
+      _.set(langs, iso3166.to2(key).toLowerCase(), value[0]), {}
+    ); // create the lang/value map
+}
+
 function getAbbreviation(properties) {
   if (properties['wof:placetype'] === 'country' && properties['wof:country']) {
     return properties['wof:country'];
@@ -167,10 +186,12 @@ function getHierarchies(id, properties) {
 */
 module.exports.create = function map_fields_stream() {
   return through2.obj(function(json_object, enc, callback) {
+    const default_names = getName(json_object.properties);
     var record = {
       id: json_object.id,
-      name: getName(json_object.properties),
+      name: default_names,
       name_aliases: getNameAliases(json_object.properties),
+      name_langs: getMultiLangNames(default_names, json_object.properties),
       abbreviation: getAbbreviation(json_object.properties),
       place_type: json_object.properties['wof:placetype'],
       lat: getLat(json_object.properties),

diff --git a/src/peliasDocGenerators.js b/src/peliasDocGenerators.js
@@ -73,10 +73,17 @@ function setupDocument(record, hierarchy) {
     }
 
     // index name aliases for all other records (where available)
-    else if (record.name_aliases.length) {
-      record.name_aliases.forEach(alias => {
-        wofDoc.setNameAlias('default', alias);
-      });
+    else {
+      if (record.name_aliases.length) {
+        record.name_aliases.forEach(alias => {
+          wofDoc.setNameAlias('default', alias);
+        });
+      }
+      if (record.name_langs) {
+        for (let lang in record.name_langs) {
+          wofDoc.setName(lang, record.name_langs[lang]);
+        }
+      }
     }
   }
   wofDoc.setCentroid({ lat: record.lat, lon: record.lon });

diff --git a/test/components/extractFieldsTest.js b/test/components/extractFieldsTest.js
@@ -54,6 +54,7 @@ tape('readStreamComponents', function(test) {
         id: 12345,
         name: 'name 1',
         name_aliases: [],
+        name_langs: {},
         place_type: 'place type 1',
         lat: 12.121212,
         lon: 21.212121,
@@ -92,6 +93,7 @@ tape('readStreamComponents', function(test) {
         id: 23456,
         name: undefined,
         name_aliases: [],
+        name_langs: {},
         place_type: undefined,
         lat: undefined,
         lon: undefined,
@@ -130,6 +132,7 @@ tape('readStreamComponents', function(test) {
         id: 12345,
         name: 'name 1',
         name_aliases: [],
+        name_langs: {},
         place_type: 'place type 1',
         lat: 12.121212,
         lon: 21.212121,
@@ -504,6 +507,7 @@ tape('readStreamComponents', function(test) {
         id: 12345,
         name: 'wof:name value',
         name_aliases: [],
+        name_langs: {},
         place_type: 'country',
         lat: undefined,
         lon: undefined,

diff --git a/test/readStreamTest.js b/test/readStreamTest.js
@@ -79,6 +79,7 @@ tape('readStream', (test) => {
             id: 123,
             name: 'name 1',
             name_aliases: [],
+            name_langs: {},
             place_type: 'place type 1',
             lat: 12.121212,
             lon: 21.212121,
@@ -94,6 +95,7 @@ tape('readStream', (test) => {
             id: 456,
             name: 'name 2',
             name_aliases: [],
+            name_langs: {},
             place_type: 'place type 2',
             lat: 13.131313,
             lon: 31.313131,
@@ -234,6 +236,7 @@ tape('readStream', (test) => {
               id: 421302191,
               name: 'name 421302191',
               name_aliases: [],
+              name_langs: {},
               abbreviation: undefined,
               place_type: undefined,
               lat: 45.240295,