feat: script to filter and generate labels (with translations)

data/categories/filter_categories.py

@@ -1,7 +1,19 @@
+"""
+https://static.openfoodfacts.org/data/taxonomies/categories.full.json
+
+How-to run ?
+> pip install openfoodfacts
+> python data/categories/filter_categories.py
+"""
+
 import json
 from openfoodfacts.taxonomy import get_taxonomy
 
 
+OFF_TAXONOMY_NAME = "category"
+OP_LANGUAGES_FILE = "src/i18n/data/languages.json"
+OLD_CATEGORIES_FILE = "src/data/category-tags.json"
+
 PARENT_CATEGORIES_ID = [
     "en:vegetables",  # 391
     "en:fruits",  # 287
@@ -26,12 +38,12 @@
 
 
 def get_languages():
-    with open("src/i18n/data/languages.json") as f:
+    with open(OP_LANGUAGES_FILE) as f:
         return json.load(f)
 
 
 def get_category_taxonomy():
-    return get_taxonomy("category")
+    return get_taxonomy(OFF_TAXONOMY_NAME)
 
 
 def get_taxonomy_node_by_id(taxonomy, node_id):
@@ -113,7 +125,7 @@ def write_categories_to_files(categories):
 
 
 def compare_new_categories_with_old_categories():
-    with open("src/data/category-tags.json") as f:
+    with open(OLD_CATEGORIES_FILE) as f:
         old_categories = json.load(f)
     print("old_categories", len(old_categories))
 
@@ -140,11 +152,6 @@ def compare_new_categories_with_old_categories():
 
 
 if __name__ == "__main__":
-    """
-    How-to run ?
-    > pip install openfoodfacts
-    > python data/categories/filter_categories.py
-    """
     # init
     CATEGORIES_FULL = get_category_taxonomy()
     print("Total number of categories:", len(CATEGORIES_FULL))

data/labels/generate_labels_translations.py

@@ -0,0 +1,126 @@
+"""
+https://static.openfoodfacts.org/data/taxonomies/labels.full.json
+
+How-to run ?
+> pip install openfoodfacts
+> python data/labels/generate_labels_translations.py
+"""
+
+import json
+from openfoodfacts.taxonomy import get_taxonomy
+
+
+OFF_TAXONOMY_NAME = "label"
+OP_LANGUAGES_FILE = "src/i18n/data/languages.json"
+OLD_LABELS_FILE = "src/data/labels-tags.json"
+
+KEEP_ONLY = [
+    "en:organic"
+]
+
+
+def get_languages():
+    with open(OP_LANGUAGES_FILE) as f:
+        return json.load(f)
+
+
+def get_label_taxonomy():
+    return get_taxonomy(OFF_TAXONOMY_NAME)
+
+
+def get_taxonomy_node_by_id(taxonomy, node_id):
+    return next((node for node in taxonomy.iter_nodes() if node.id == node_id), None)
+
+
+def get_taxonomy_node_list_by_id_list(taxonomy, node_id_list):
+    node_list = list()
+    for node_id in node_id_list:
+        taxonomy_node = get_taxonomy_node_by_id(taxonomy, node_id)
+        if taxonomy_node:
+            node_list.append(taxonomy_node)
+    return node_list
+
+
+def taxonomy_node_list_to_dict_list(node_list, delete_parents=False):
+    node_dict_list = list()
+    for node in node_list:
+        node_dict = { "id": node.id, **node.to_dict() }
+        if delete_parents:
+            del node_dict["parents"]
+        node_dict_list.append(node_dict)
+    return node_dict_list
+
+
+def get_taxonomy_node_children_full_list(taxonomy, node_parent):
+    children_node_list = list()
+    for node in taxonomy.iter_nodes():
+        node_parents = node.get_parents_hierarchy()
+        if next((n for n in node_parents if n == node_parent), None):
+            children_node_list.append(node)
+    return children_node_list
+
+
+def filter_labels(taxonomy):
+    node_list = list()
+    for node in taxonomy.iter_nodes():
+        if node.id in KEEP_ONLY:
+            node_list.append(node)
+    return node_list
+
+
+def write_labels_to_files(labels):
+    languages = get_languages()
+    for language in languages:
+        language_code = language['code']
+        language_labels = list()
+        # for each label, get translation (or default to en)
+        for label in labels:
+            language_label_name = label['name'][language_code] if (language_code in label['name']) else label['name']['en']
+            language_labels.append({"id": label['id'], "name": language_label_name})
+        # order by name
+        language_labels = sorted(language_labels, key=lambda x: x['name'])
+        # write to file
+        with open(f"src/data/labels/{language['code']}.json", "w") as f:
+            json.dump(language_labels, f, ensure_ascii=False)
+
+
+def compare_new_labels_with_old_labels():
+    with open("src/data/labels-tags.json") as f:
+        old_labels = json.load(f)
+    print("old_labels", len(old_labels))
+
+    with open("src/data/labels/en.json") as f:
+        new_labels = json.load(f)
+    print("new_labels", len(new_labels))
+
+    # check missing in new
+    label_missing_in_new_list = list()
+    for label in old_labels:
+        found = next((c for c in new_labels if c['id'] == label['id']), None)
+        if not found:
+            label_missing_in_new_list.append(label)
+    print("missing in new", len(label_missing_in_new_list))
+    print(label_missing_in_new_list)
+
+    # check missing in old
+    label_missing_in_old_list = list()
+    for label in old_labels:
+        found = next((c for c in old_labels if c['id'] == label['id']), None)
+        if not found:
+            label_missing_in_old_list.append(label)
+    print("missing in old", len(label_missing_in_old_list))
+
+
+if __name__ == "__main__":
+    # init
+    LABELS_FULL = get_label_taxonomy()
+    print("Total number of labels:", len(LABELS_FULL))
+
+    labels_filtered = filter_labels(LABELS_FULL)
+    labels_filtered_to_dict_list = taxonomy_node_list_to_dict_list(list(labels_filtered), delete_parents=True)
+    print("Labels remaining:", len(labels_filtered_to_dict_list))
+
+    write_labels_to_files(labels_filtered_to_dict_list)
+    print("Wrote to language files")
+
+    compare_new_labels_with_old_labels()

data/origins/generate_origins_translations.py

@@ -1,7 +1,17 @@
+"""
+https://static.openfoodfacts.org/data/taxonomies/origins.full.json
+
+How-to run ?
+> pip install openfoodfacts
+> python data/origins/generate_origins_translations.py
+"""
+
 import json
 from openfoodfacts.taxonomy import get_taxonomy
 
 
+OFF_TAXONOMY_NAME = "origin"
+OP_LANGUAGES_FILE = "src/i18n/data/languages.json"
 OLD_ORIGINS_FILE = "src/data/origins-tags.json"
 
 WITH_PROPERTIES = [
@@ -25,12 +35,12 @@
 
 
 def get_languages():
-    with open("src/i18n/data/languages.json") as f:
+    with open(OP_LANGUAGES_FILE) as f:
         return json.load(f)
 
 
 def get_origin_taxonomy():
-    return get_taxonomy("origin")
+    return get_taxonomy(OFF_TAXONOMY_NAME)
 
 
 def get_taxonomy_node_by_id(taxonomy, node_id):
@@ -119,11 +129,6 @@ def compare_new_origins_with_old_origins():
 
 
 if __name__ == "__main__":
-    """
-    How-to run ?
-    > pip install openfoodfacts
-    > python data/origins/generate_origins_translations.py
-    """
     # init
     ORIGINS_FULL = get_origin_taxonomy()
     print("Total number of origins:", len(ORIGINS_FULL))

src/data/labels/aa.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/ach.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/af.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/ak.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/am.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/ar.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/as.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/ast.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/az.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/be.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/ber.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/bg.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Био"}]

src/data/labels/bm.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/bn.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/bo.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/br.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/bs.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/ca.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Orgànic"}]

src/data/labels/ce.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/chr.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/co.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/crs.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/cs.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Bio"}]

src/data/labels/cv.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/cy.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/da.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Økologisk"}]

src/data/labels/de.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Bio"}]

src/data/labels/el.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/en.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/en_AU.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/en_GB.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/eo.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/es.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Ecológico"}]

src/data/labels/et.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/eu.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/fa.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/fi.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Luomu"}]

src/data/labels/fil.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/fo.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/fr.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Bio"}]

src/data/labels/ga.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/gd.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/gl.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/gu.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/ha.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/he.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "אורגני"}]

src/data/labels/hi.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/hr.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Ekološki uzgoj"}]

src/data/labels/ht.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/hu.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Bio"}]

src/data/labels/hy.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/id.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/ii.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/is.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/it.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Biologico"}]

src/data/labels/iu.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]

src/data/labels/ja.json

@@ -0,0 +1 @@
+[{"id": "en:organic", "name": "Organic"}]