fix tokenizer_info when convert the model (#661)

InternLM · Nov 8, 2023 · 9febf61 · 9febf61
1 parent 529e56b
commit 9febf61
Showing 1 changed file with 5 additions and 13 deletions.
diff --git a/lmdeploy/turbomind/deploy/source_model/llama.py b/lmdeploy/turbomind/deploy/source_model/llama.py
@@ -5,7 +5,6 @@
 
 import torch
 from safetensors.torch import load_file
-from sentencepiece import SentencePieceProcessor
 
 from lmdeploy.tokenizer import Tokenizer
 
@@ -168,18 +167,11 @@ def get_mgrs(self):
 
     def tokenizer_info(self):
         """Read tokenizer info."""
-        assert osp.isfile(self.tokenizer_path), self.tokenizer_path
-        try:
-            tk_model = SentencePieceProcessor(model_file=self.tokenizer_path)
-            # BOS / EOS token IDs
-            n_words = tk_model.vocab_size
-            bos_id = tk_model.bos_token_id
-            eos_id = tk_model.eos_token_id
-        except Exception:
-            tk_model = Tokenizer(self.model_path)
-            n_words = tk_model.vocab_size
-            bos_id = tk_model.bos_token_id
-            eos_id = tk_model.eos_token_id
+        assert osp.isdir(self.model_path), self.model_path
+        tk_model = Tokenizer(self.model_path)
+        n_words = tk_model.vocab_size
+        bos_id = tk_model.bos_token_id
+        eos_id = tk_model.eos_token_id
         return n_words, bos_id, eos_id
 
     def model_info(self):