feat: use gzipped version of fallback taxonomy files

And fix encoding UTF-8 errors in ingredient taxonomy JSON
openfoodfacts · Oct 11, 2022 · 813cb41 · 813cb41
1 parent 67112f9
commit 813cb41
Show file tree

Hide file tree

Showing 10 changed files with 6 additions and 13 deletions.
diff --git a/data/taxonomies/brands.full.json b/data/taxonomies/brands.full.json
diff --git a/data/taxonomies/brands.full.json.gz b/data/taxonomies/brands.full.json.gz
diff --git a/data/taxonomies/categories.full.json b/data/taxonomies/categories.full.json
diff --git a/data/taxonomies/categories.full.json.gz b/data/taxonomies/categories.full.json.gz
diff --git a/data/taxonomies/ingredients.full.json b/data/taxonomies/ingredients.full.json
diff --git a/data/taxonomies/ingredients.full.json.gz b/data/taxonomies/ingredients.full.json.gz
diff --git a/data/taxonomies/labels.full.json b/data/taxonomies/labels.full.json
diff --git a/data/taxonomies/labels.full.json.gz b/data/taxonomies/labels.full.json.gz
diff --git a/robotoff/settings.py b/robotoff/settings.py
@@ -129,10 +129,10 @@ def off_credentials() -> Dict[str, str]:
 # Taxonomies are huge JSON files that describe many concepts in OFF, in many languages, with synonyms. Those are the full version of taxos.
 
 TAXONOMY_DIR = DATA_DIR / "taxonomies"
-TAXONOMY_CATEGORY_PATH = TAXONOMY_DIR / "categories.full.json"
-TAXONOMY_INGREDIENT_PATH = TAXONOMY_DIR / "ingredients.full.json"
-TAXONOMY_LABEL_PATH = TAXONOMY_DIR / "labels.full.json"
-TAXONOMY_BRAND_PATH = TAXONOMY_DIR / "brands.full.json"
+TAXONOMY_CATEGORY_PATH = TAXONOMY_DIR / "categories.full.json.gz"
+TAXONOMY_INGREDIENT_PATH = TAXONOMY_DIR / "ingredients.full.json.gz"
+TAXONOMY_LABEL_PATH = TAXONOMY_DIR / "labels.full.json.gz"
+TAXONOMY_BRAND_PATH = TAXONOMY_DIR / "brands.full.json.gz"
 INGREDIENTS_FR_PATH = TAXONOMY_DIR / "ingredients_fr.txt"
 INGREDIENT_TOKENS_PATH = TAXONOMY_DIR / "ingredients_tokens.txt"
 FR_TOKENS_PATH = TAXONOMY_DIR / "fr_tokens_lower.gz"

diff --git a/robotoff/taxonomy.py b/robotoff/taxonomy.py
@@ -4,11 +4,10 @@
 from enum import Enum
 from typing import Dict, Iterable, List, Optional, Set, Union
 
-import orjson
 import requests
 
 from robotoff import settings
-from robotoff.utils import get_logger, http_session
+from robotoff.utils import get_logger, http_session, load_json
 from robotoff.utils.cache import CachedStore
 from robotoff.utils.types import JSONType
 
@@ -223,9 +222,7 @@ def from_dict(cls, data: JSONType) -> "Taxonomy":
 
     @classmethod
     def from_json(cls, file_path: Union[str, pathlib.Path]):
-        with open(str(file_path), "rb") as f:
-            data = orjson.loads(f.read())
-            return cls.from_dict(data)
+        return cls.from_dict(load_json(file_path, compressed=True))
 
     def to_graph(self):
         """Generate a networkx.DiGraph from the taxonomy."""