hotfix for get_tokenizer (#704)

alibaba · Sep 20, 2023 · 8da9f9f · 8da9f9f
1 parent b963f62
commit 8da9f9f
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/federatedscope/llm/dataloader/dataloader.py b/federatedscope/llm/dataloader/dataloader.py
@@ -62,7 +62,7 @@ def __call__(self, instances):
         )
 
 
-def get_tokenizer(model_name, cache_dir, tok_len=128, pkg='huggingface'):
+def get_tokenizer(model_name, cache_dir, tok_len=128, pkg='huggingface_llm'):
     """
     This function loads a tokenizer from a pretrained model name and adds some
     default special tokens if they are not already defined. It also sets the

diff --git a/federatedscope/llm/misc/fschat.py b/federatedscope/llm/misc/fschat.py
@@ -48,9 +48,9 @@ def __init__(self, config):
             config: A FS configuration object that contains various settings
                 for the chatbot.
         """
-        model_name, _ = config.model.type.split('@')
+        model_name, model_hub = config.model.type.split('@')
         self.tokenizer, _ = get_tokenizer(model_name, config.data.root,
-                                          config.llm.tok_len)
+                                          config.llm.tok_len, model_hub)
         self.model = get_llm(config)
 
         self.device = f'cuda:{config.device}'