fix(language_field_trimming): bugfix for #135

pelias · Jul 20, 2021 · 00b3fb7 · 00b3fb7
1 parent f949386
commit 00b3fb7
Show file tree

Hide file tree

Showing 2 changed files with 64 additions and 6 deletions.
diff --git a/post/language_field_trimming.js b/post/language_field_trimming.js
@@ -27,23 +27,37 @@ function deduplication(doc) {
 
     // fetch the 'default' language
     var defaults = _.get(field, 'default');
-    if (!_.isArray(defaults) || _.isEmpty(defaults)) { return; }
+
+    // no default names, nothing to do; continue
+    if (_.isEmpty(defaults)) { return; }
+
+    // convert scalar values to arrays
+    defaults = _.castArray(defaults);
 
     // iterate over other languages in the field
     _.each(field, (names, lang) => {
 
       // skip the 'default' language
       if (lang === 'default'){ return; }
 
+      // no names, nothing to do; continue
+      if (_.isEmpty(names)) { return; }
+
+      // convert scalar values to arrays
+      names = _.castArray(names);
+
       // filter entries from this language which appear in the 'default' lang
-      if (_.isArray(names) || !_.isEmpty(names)) {
-        field[lang] = _.difference(names, defaults);
-      }
+      field[lang] = _.difference(names, defaults);
 
       // clean up empty language arrays
       if (_.isEmpty(field[lang])) {
         delete field[lang];
       }
+
+      // flatten single-value arrays
+      else if(_.size(field[lang]) === 1) {
+        field[lang] = _.first(field[lang]);
+      }
     });
   });
 }

diff --git a/test/post/language_field_trimming.js b/test/post/language_field_trimming.js
@@ -22,7 +22,7 @@ module.exports.tests.dedupe = function (test) {
     language_field_trimming(doc);
 
     t.deepEquals(doc.name.default, ['test1', 'test2', 'test3']);
-    t.deepEquals(doc.name.en, ['test4']);
+    t.deepEquals(doc.name.en, 'test4');
     t.false(doc.name.de);
 
     t.end();
@@ -45,11 +45,55 @@ module.exports.tests.dedupe = function (test) {
     language_field_trimming(doc);
 
     t.deepEquals(doc.phrase.default, ['test1', 'test2', 'test3']);
-    t.deepEquals(doc.phrase.en, ['test4']);
+    t.deepEquals(doc.phrase.en, 'test4');
     t.false(doc.phrase.de);
 
     t.end();
   });
+
+  test('dedupe - two default names, one from a language code', function (t) {
+    var doc = new Document('mysource', 'mylayer', 'myid');
+
+    doc.setName('default', 'test1');
+    doc.setNameAlias('default', 'test2');
+    doc.setName('ru', 'test3');
+
+    language_field_trimming(doc);
+
+    t.deepEquals(doc.name.default, ['test1', 'test2']);
+    t.deepEquals(doc.name.ru, 'test3');
+
+    t.end();
+  });
+
+  test('dedupe - one default name, two from a language code', function (t) {
+    var doc = new Document('mysource', 'mylayer', 'myid');
+
+    doc.setName('default', 'test1');
+    doc.setName('ru', 'test2');
+    doc.setNameAlias('ru', 'test3');
+
+    language_field_trimming(doc);
+
+    t.deepEquals(doc.name.default, 'test1');
+    t.deepEquals(doc.name.ru, ['test2', 'test3']);
+
+    t.end();
+  });
+
+  test('dedupe - zero default names, two from a language code', function (t) {
+    var doc = new Document('mysource', 'mylayer', 'myid');
+
+    doc.setName('ru', 'test1');
+    doc.setNameAlias('ru', 'test2');
+
+    language_field_trimming(doc);
+
+    t.false(doc.name.default);
+    t.deepEquals(doc.name.ru, ['test1', 'test2']);
+
+    t.end();
+  });
 };
 
 module.exports.all = function (tape, common) {