feat(mouthing): add mouthing generation

pyproject.toml

@@ -7,7 +7,7 @@ authors = [
 ]
 readme = "README.md"
 dependencies = [
-    "Pillow",
+    "Pillow"
 ]
 
 [project.optional-dependencies]
@@ -16,6 +16,11 @@ dev = [
     "pylint",
     "numpy", # to test visualizer
 ]
+mouthing = [
+    # For IPA transliteration
+    "epitran",
+    "g2pk"
+]
 
 [tool.yapf]
 based_on_style = "google"

signwriting/fingerspelling/fingerspelling.py

@@ -4,11 +4,27 @@
 from pathlib import Path
 from typing import Union
 
-from signwriting.utils.join_signs import join_signs
+from signwriting.utils.join_signs import join_signs_vertical, join_signs_horizontal
 
 FINGERSPELLING_DIR = Path(__file__).parent / "data"
 
 
+@functools.lru_cache(maxsize=None)
+def get_chars_by(value: str, category: str):
+    categories = ["LANGUAGE", "COUNTRY", "SIGNED", "NAME"]
+    if category not in categories:
+        raise ValueError(f"Category must be one of {categories}")
+    category_index = categories.index(category)
+
+    # iterate over the directory
+    for file in FINGERSPELLING_DIR.iterdir():
+        file_category = file.stem.split("-")[category_index]
+        if file_category == value:
+            return get_chars(file.stem)
+
+    raise ValueError(f"Could not find a file with {category} {value}")
+
+
 @functools.lru_cache(maxsize=None)
 def get_chars(language: str):
     with open(FINGERSPELLING_DIR / f"{language}.txt", "r", encoding="utf-8") as f:
@@ -17,27 +33,29 @@ def get_chars(language: str):
     return {first.lower(): others for [first, *others] in lines}
 
 
-def spell(characters: str, language=None, chars=None) -> Union[str, None]:
+def spell(word: str, language=None, chars=None, vertical=True) -> Union[str, None]:
     if chars is None:
         if language is None:
             raise ValueError("Either language or chars must be provided")
         chars = get_chars(language)
 
     sl = []
     caret = 0
-    while caret < len(characters):
+    while caret < len(word):
         found = False
         for c, options in chars.items():
-            if characters[caret:caret + len(c)].lower() == c:
+            if word[caret:caret + len(c)].lower() == c:
                 sl.append(random.choice(options))
                 caret += len(c)
                 found = True
                 break
         if not found:
             return None
-    return join_signs(*sl, spacing=5)
+    if vertical:
+        return join_signs_vertical(*sl, spacing=5)
+    return join_signs_horizontal(*sl, spacing=5)
 
 
 if __name__ == "__main__":
-    for word in ["12345", "hello", "Amit"]:
-        print(word, spell(word, language='en-us-ase-asl'))
+    for _word in ["12345", "hello", "Amit"]:
+        print(_word, spell(_word, language='en-us-ase-asl', vertical=False))

signwriting/mouthing/README.md

@@ -86,3 +86,19 @@ Find anything wrong or missing? Please help improve this resource by submitting
 | a͡ʊ | ![a͡ʊ](standard/a͡ʊ.png) | ![M531x518S34c00469x483S34d00495x483](https://www.signbank.org/signpuddle2.0/glyphogram.php?text=M531x518S34c00469x483S34d00495x483&pad=10&size=2) | au, ao | Auto, Stau, laut, Haus, Kakao | „ow” as in English „cow” but in a more brief and clipped manner | Start with your tongue low and your mouth open, then move your tongue upward and close your lips into a rounded shape while voicing. |
 | ɔ͡ø | ![ɔ͡ø](standard/ɔ͡ø.png) | ![M531x518S34900469x483S34800495x483](https://www.signbank.org/signpuddle2.0/glyphogram.php?text=M531x518S34900469x483S34800495x483&pad=10&size=2) | äu, eu, oi | Feuer, Eule, Gebäude, aufräumen, Konvoi | „oy“ as in English „toy“ | Start with your tongue low and your lips rounded, then move your tongue upward and forward while rounding your lips tighter. |
 | a͡ɪ | ![a͡ɪ](standard/a͡ɪ.png) | ![M531x518S34c00469x483S34800495x483](https://www.signbank.org/signpuddle2.0/glyphogram.php?text=M531x518S34c00469x483S34800495x483&pad=10&size=2) | ei, ai, ay, ey, eih | Hai, Eimer, leise, Meyer, Reihe, Karl May | English „i“ as in „high“ or „mine” | Start with your tongue low and your mouth open, then move your tongue upward and close your lips into a slight smile while voicing. |
+
+
+## Install flite
+
+To make English IPA translitaration work, you need to install `flite`:
+
+```bash
+git clone https://github.com/festvox/flite.git
+cd flite
+
+./configure && make
+sudo make install
+cd testsuite
+make lex_lookup
+sudo cp lex_lookup /usr/local/bin
+```

signwriting/mouthing/mouthing.json

@@ -11,7 +11,8 @@
     "grapheme": "a",
     "example": "Salz, Wand, Mann",
     "description": "short „u” as in Southern English „but”",
-    "instruction": "Keep your tongue in the center of your mouth, slightly raised, with your mouth open and lips relaxed while voicing."
+    "instruction": "Keep your tongue in the center of your mouth, slightly raised, with your mouth open and lips relaxed while voicing.",
+    "alternatives": ["a", "æ"]
   },
   "ɛː": {
     "writing": "M518x518S34a00482x483",
@@ -154,7 +155,7 @@
     "instruction": "Place the tip of your tongue against the ridge behind your upper front teeth, let the air pass through your nose, and voice."
   },
   "ŋ": {
-    "writing": "",
+    "writing": "M518x518S35d00482x483S33110499x490S20500503x493",
     "grapheme": "n, ng, nk",
     "example": "Ring, Zange, Junge, Bon, krank, trinken",
     "description": "like the „ng“ sound in the English word „song“ or „long”",
@@ -165,7 +166,8 @@
     "grapheme": "o, oo, oh",
     "example": "Ofen, Oma, Kohle, Zoo, Krone, groß, Obst",
     "description": "long „o” similar to „go” but with the lips more rounded and more open",
-    "instruction": "Raise the back of your tongue close to the roof of your mouth, round your lips, and voice."
+    "instruction": "Raise the back of your tongue close to the roof of your mouth, round your lips, and voice.",
+    "alternatives": ["o"]
   },
   "ɔ": {
     "writing": "M518x518S34900482x483",
@@ -277,7 +279,8 @@
     "grapheme": "u, uh",
     "example": "Buch, Tube, Huhn, Stuhl",
     "description": "long „oo” as in „boot” with the lips much more rounded",
-    "instruction": "Raise the back of your tongue high in the mouth, round your lips tightly, and voice."
+    "instruction": "Raise the back of your tongue high in the mouth, round your lips tightly, and voice.",
+    "alternatives": ["w"]
   },
   "ʊ": {
     "writing": "M518x518S34600482x483",

signwriting/mouthing/mouthing.py

@@ -0,0 +1,82 @@
+import functools
+import json
+import re
+from pathlib import Path
+from typing import Union
+
+from epitran import Epitran
+from signwriting.formats.sign_to_fsw import sign_to_fsw
+
+from signwriting.formats.fsw_to_sign import fsw_to_sign
+
+from signwriting.utils.join_signs import join_signs_horizontal, sign_from_symbols
+
+MOUTHING_INDEX = Path(__file__).parent / "mouthing.json"
+
+
+@functools.lru_cache()
+def get_mouthings():
+    with open(MOUTHING_INDEX, "r", encoding="utf-8") as f:
+        mouthings = json.load(f)
+
+    for info in list(mouthings.values()):
+        if "alternatives" in info:
+            for alternative in info["alternatives"]:
+                mouthings[alternative] = info
+
+    return mouthings
+
+
+@functools.lru_cache()
+def get_mouthings_without_aspiration():
+    mouthings = get_mouthings()
+
+    for info in mouthings.values():
+        if "S335" in info["writing"]:
+            info["writing"] = re.sub(r"S335..\d{3}x\d{3}", "", info["writing"])
+        sign = fsw_to_sign(info["writing"])
+        sign = sign_from_symbols(sign["symbols"])
+        info["writing"] = sign_to_fsw(sign)
+
+    return mouthings
+
+
+def mouth_ipa_single(word: str, aspiration=False) -> Union[str, None]:
+    mouthings = get_mouthings() if aspiration else get_mouthings_without_aspiration()
+
+    # Make sure to look at long symbols first
+    mouthings = sorted(list(mouthings.items()), key=lambda x: len(x[0]), reverse=True)
+
+    sl = []
+    caret = 0
+    while caret < len(word):
+        found = False
+        for symbol, info in mouthings:
+            if word[caret:caret + len(symbol)].lower() == symbol:
+                sl.append(info["writing"])
+                caret += len(symbol)
+                found = True
+                break
+        if not found:
+            print(f"Symbol not found: {word[caret:caret + 1]}")
+            return None
+    return join_signs_horizontal(*sl, spacing=-10)
+
+
+def mouth_ipa(characters: str, aspiration=False) -> Union[str, None]:
+    words = [mouth_ipa_single(word, aspiration=aspiration) for word in characters.split(" ")]
+    if any(word is None for word in words):
+        return None
+
+    return join_signs_horizontal(*words, spacing=10)
+
+
+def mouth(word: str, language: str, aspiration=False):
+    epi = Epitran(language, ligatures=True)
+    ipa = epi.transliterate(word)
+    return mouth_ipa(ipa, aspiration=aspiration)
+
+
+if __name__ == "__main__":
+    for _word in ["hello", "Amit", "high", "sign writing", "SignWriting"]:
+        print(_word, mouth(_word, language='eng-Latn'))

signwriting/utils/join_signs.py

@@ -1,33 +1,87 @@
+from collections import namedtuple
+
 from signwriting.formats.fsw_to_sign import fsw_to_sign
 from signwriting.formats.sign_to_fsw import sign_to_fsw
-from signwriting.types import Sign
+from signwriting.types import Sign, SignSymbol
+from signwriting.visualizer.visualize import get_symbol_size
 
 
-def all_ys(_sign):
-    return [s["position"][1] for s in _sign["symbols"]]
+def all_axis(_sign, axis):
+    axis_index = 0 if axis == "x" else 1
+    return [s["position"][axis_index] for s in _sign["symbols"]]
 
 
-def join_signs(*fsws: str, spacing: int = 0):
+def init_join(*fsws: str):
     signs = [fsw_to_sign(fsw) for fsw in fsws]
-    new_sign: Sign = {"box": {"symbol": "M", "position": (500, 500)}, "symbols": []}
+    return [sign for sign in signs if len(sign["symbols"]) > 0]
+
 
+def join_signs_vertical(*fsws: str, spacing: int = 0):
+    signs = init_join(*fsws)
+    symbols = []
     accumulative_offset = 0
 
     for sign in signs:
-        sign_min_y = min(all_ys(sign))
+        sign_min_y = min(all_axis(sign, "y"))
         sign_offset_y = accumulative_offset + spacing - sign_min_y
         accumulative_offset += (sign["box"]["position"][1] - sign_min_y) + spacing  # * 2
 
-        new_sign["symbols"] += [{
-            "symbol": s["symbol"],
-            "position": (s["position"][0], s["position"][1] + sign_offset_y)
-        } for s in sign["symbols"]]
+        for symbol in sign["symbols"]:
+            symbols.append({
+                "symbol": symbol["symbol"],
+                "position": (symbol["position"][0], symbol["position"][1] + sign_offset_y)
+            })
+
+    new_sign = sign_from_symbols(symbols, fix_x=False)
+    return sign_to_fsw(new_sign)
+
+
+def join_signs_horizontal(*fsws: str, spacing: int = 0):
+    signs = init_join(*fsws)
+    symbols = []
+    accumulative_offset = 0
 
-    # Recenter around box center
-    sign_middle = max(all_ys(new_sign)) // 2
+    for sign in signs:
+        sign_min_x = min(all_axis(sign, "x"))
+        sign_offset_x = accumulative_offset + spacing - sign_min_x
+        accumulative_offset += (sign["box"]["position"][0] - sign_min_x) + spacing  # * 2
 
-    for symbol in new_sign["symbols"]:
-        symbol["position"] = (symbol["position"][0],
-                              new_sign["box"]["position"][1] - sign_middle + symbol["position"][1])
+        for symbol in sign["symbols"]:
+            symbols.append({
+                "symbol": symbol["symbol"],
+                "position": (symbol["position"][0] + sign_offset_x, symbol["position"][1])
+            })
 
+    new_sign = sign_from_symbols(symbols, fix_y=False)
     return sign_to_fsw(new_sign)
+
+
+Point = namedtuple("Point", ["x", "y"])
+
+
+def sign_from_symbols(symbols: list[SignSymbol], fix_x=True, fix_y=True) -> Sign:
+    min_p = Point(x=999, y=999)
+    max_p = Point(x=0, y=0)
+    for symbol in symbols:
+        min_p = Point(x=min(min_p.x, symbol["position"][0]),
+                      y=min(min_p.y, symbol["position"][1]))
+
+        symbol_width, symbol_height = get_symbol_size(symbol["symbol"])
+        max_p = Point(x=max(max_p.x, symbol["position"][0] + symbol_width),
+                      y=max(max_p.y, symbol["position"][1] + symbol_height))
+
+    box_p = Point(x=500 + (max_p.x - min_p.x) // 2,
+                  y=500 + (max_p.y - min_p.y) // 2)
+    box = {"symbol": "M", "position": box_p}
+    size = Point(x=max_p.x - min_p.x,
+                 y=max_p.y - min_p.y)
+
+    for symbol in symbols:
+        symbol_x, symbol_y = symbol["position"]
+        if fix_x:
+            symbol_x += box_p.x - min_p.x - size.x
+        if fix_y:
+            symbol_y += box_p.y - min_p.y - size.y
+        symbol["position"] = (symbol_x, symbol_y)
+
+    return {"box": box, "symbols": symbols}

signwriting/utils/test_join_signs.py

@@ -1,30 +1,34 @@
 import unittest
 
-from signwriting.utils.join_signs import join_signs
+from signwriting.utils.join_signs import join_signs_vertical
 
 
 class JoinSignsCase(unittest.TestCase):
 
     def test_join_two_characters(self):
         char_a = 'M507x507S1f720487x492'
         char_b = 'M507x507S14720493x485'
-        result_sign = join_signs(char_a, char_b)
-        self.assertEqual(result_sign, 'M500x500S1f720487x493S14720493x508')
+        result_sign = join_signs_vertical(char_a, char_b)
+        self.assertEqual('M510x518S1f720487x481S14720493x496', result_sign)
 
     def test_join_alphabet_characters(self):
         chars = [
             "M510x508S1f720490x493", "M507x511S14720493x489", "M509x510S16d20492x490", "M508x515S10120492x485",
             "M508x508S14a20493x493", "M511x515S1ce20489x485", "M515x508S1f000486x493", "M515x508S11502485x493",
             "M511x510S19220490x491", "M519x518S19220498x499S2a20c482x483"
         ]
-        result_sign = join_signs(*chars, spacing=10)
+        result_sign = join_signs_vertical(*chars, spacing=10)
         # pylint: disable=line-too-long
         self.assertEqual(
-            result_sign,
-            'M500x500S1f720490x362S14720493x387S16d20492x419S10120492x449S14a20493x489S1ce20489x514S1f000486x554S11502485x579S19220490x604S19220498x649S2a20c482x633'
-            # noqa: E501
+            'M518x653S1f720490x347S14720493x372S16d20492x404S10120492x434S14a20493x474S1ce20489x499S1f000486x539S11502485x564S19220490x589S19220498x634S2a20c482x618',
+            result_sign
         )
 
+    def test_join_empty_sign(self):
+        char_a = 'M507x507S1f720487x492'
+        char_b = 'M507x507'
+        result_sign = join_signs_vertical(char_a, char_b)
+        self.assertEqual('M510x507S1f720487x492', result_sign)
 
 if __name__ == '__main__':
     unittest.main()