Test languages exemplars canonical duplicates

tests/test_data_languages.py

@@ -15,7 +15,8 @@
 # limitations under the License.
 #
 import pytest
-from collections import Counter
+import unicodedata
+from collections import defaultdict, Counter
 from gflanguages import LoadLanguages
 
 
@@ -30,3 +31,21 @@ def test_languages_exemplars_duplicates(exemplar_name):
         counts = sorted(counter.most_common(), key=lambda pair:
                         exemplar.index(pair[0]))
         assert (counts == [(v, 1) for v in exemplar])
+
+
+@pytest.mark.parametrize(
+    "exemplar_name",
+    ["base", "auxiliary", "marks", "numerals", "punctuation", "index"]
+)
+def test_languages_exemplars_canonical_duplicates(exemplar_name):
+    for code, lang in LoadLanguages().items():
+        exemplar = getattr(lang.exemplar_chars, exemplar_name).split()
+        normalized = defaultdict(set)
+
+        for g in exemplar:
+            if g[0] == "{" and g[-1] == "}":
+                g = g.lstrip("{").rstrip("}")
+            normalized[unicodedata.normalize("NFC", g)].add(g)
+
+        for n, gs in normalized.items():
+            assert len(gs) == 1