Glue words (WIP)

addok_france/__init__.py

@@ -14,6 +14,7 @@
 extract_address = yielder(utils.extract_address)
 glue_ordinal = utils.glue_ordinal
 fold_ordinal = yielder(utils.fold_ordinal)
+glue_words = utils.glue_words
 flag_housenumber = utils.flag_housenumber
 make_labels = utils.make_labels
 remove_leading_zeros = yielder(utils.remove_leading_zeros)

addok_france/utils.py

@@ -127,6 +127,16 @@ def fold_ordinal(s):
     return s
 
 
+GLUE_WORDS = ["mont", "val", "le", "la", "l", "champ"]
+
+def glue_words(tokens):
+    """ glue 'MONT GRIFFON' into 'MONTGRIFFON' """
+    for _, token, next_ in neighborhood(tokens):
+        yield token
+        if token in GLUE_WORDS and next_ and next_.isalpha() and len(next_)>2:
+            yield token.update(token+next_)
+
+
 def remove_leading_zeros(s):
     """0003 => 3."""
     # Limit digits from 1 to 3 in order to avoid processing postcodes.

tests/test_utils.py

@@ -8,7 +8,7 @@
 from addok.helpers.text import Token
 from addok_france.utils import (clean_query, extract_address, flag_housenumber,
                                 fold_ordinal, glue_ordinal, make_labels,
-                                remove_leading_zeros)
+                                remove_leading_zeros, glue_words)
 
 
 @pytest.mark.parametrize("input,expected", [
@@ -331,3 +331,15 @@ def test_make_municipality_labels(config):
         '59000 Lille',
         'Lille 59000',
     ]
+
+
+@pytest.mark.parametrize("inputs,expected", [
+    (['mont', 'griffon'], ['mont', 'montgriffon', 'griffon']),
+    (['champ', 'vallon'], ['champ', 'champvallon', 'vallon']),
+    (['val', 'suzon'], ['val', 'valsuzon', 'suzon']),
+    (['l', 'a', 'peu', 'pres'], ['l', 'a', 'peu', 'pres']),
+    (['l', 'un', 'des'], ['l', 'un', 'des']),
+])
+def test_glue_ordinal(inputs, expected):
+    tokens = [Token(input_) for input_ in inputs]
+    assert list(glue_words(tokens)) == expected