From 4e9ca9831b210ac7342e5b44456d1ba75c119d5a Mon Sep 17 00:00:00 2001 From: Frank Liu Date: Sun, 1 Oct 2023 12:47:22 +0800 Subject: [PATCH] [tokenizer] Allows import non-english model --- extensions/tokenizers/src/main/python/huggingface_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/tokenizers/src/main/python/huggingface_models.py b/extensions/tokenizers/src/main/python/huggingface_models.py index 3418815d5c4..7e5ada55ea9 100644 --- a/extensions/tokenizers/src/main/python/huggingface_models.py +++ b/extensions/tokenizers/src/main/python/huggingface_models.py @@ -83,7 +83,7 @@ def list_models(self, args: Namespace) -> List[dict]: continue languages = get_lang_tags(model_info) - if "en" not in languages: + if "en" not in languages and not os.environ["HF_IMPORT_ALL"]: logging.warning(f"Skip non-English model: {model_id}.") continue