Fix chatglm tokenizer failed when transformers>=4.45.0 (#2520)

* Fix chatglm tokenizer failed when transformers>=4.45.0 * fix chatglm2-6b
InternLM · Sep 26, 2024 · bb1dfa6 · bb1dfa6
1 parent 0323103
commit bb1dfa6
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 2 deletions.
diff --git a/lmdeploy/tokenizer.py b/lmdeploy/tokenizer.py
@@ -519,6 +519,15 @@ class ChatGLM4Tokenizer(HuggingFaceTokenizer):
 
     def __init__(self, model_path):
         super(ChatGLM4Tokenizer, self).__init__(model_path)
+        original_pad = self.model._pad
+
+        def __pad(*args, **kwargs):
+            if 'padding_side' in kwargs:
+                kwargs.pop('padding_side')
+            return original_pad(*args, **kwargs)
+
+        # fix for transformers>4.45.0
+        self.model._pad = __pad
 
     def encode(self,
                s: str,
@@ -534,6 +543,22 @@ def encode(self,
                                                      **kwargs)
 
 
+class ChatGLMTokenizer(HuggingFaceTokenizer):
+    """tokenizer of GLM2."""
+
+    def __init__(self, model_path):
+        super(ChatGLMTokenizer, self).__init__(model_path)
+        original_pad = self.model._pad
+
+        def __pad(*args, **kwargs):
+            if 'padding_side' in kwargs:
+                kwargs.pop('padding_side')
+            return original_pad(*args, **kwargs)
+
+        # fix for transformers>4.45.0
+        self.model._pad = __pad
+
+
 class Tokenizer:
     """Tokenize prompts or de-tokenize tokens into texts.
 
@@ -563,6 +588,8 @@ def __init__(self, model_file: str):
             config_tokenizer_class = tokenizer_config.get('tokenizer_class')
             if config_tokenizer_class == 'ChatGLM4Tokenizer':
                 self.model = ChatGLM4Tokenizer(model_folder)
+            elif config_tokenizer_class == 'ChatGLMTokenizer':
+                self.model = ChatGLMTokenizer(model_folder)
             else:
                 self.model = HuggingFaceTokenizer(model_folder)
 

diff --git a/tests/test_lmdeploy/test_tokenizer.py b/tests/test_lmdeploy/test_tokenizer.py
@@ -2,7 +2,7 @@
 
 import pytest
 
-from lmdeploy.tokenizer import DetokenizeState, HuggingFaceTokenizer
+from lmdeploy.tokenizer import DetokenizeState, HuggingFaceTokenizer, Tokenizer
 
 
 @pytest.mark.parametrize('model_path', [
@@ -20,7 +20,7 @@
 @pytest.mark.parametrize('skip_special_tokens', [True, False])
 def test_tokenizer(model_path, input, interval, add_special_tokens,
                    skip_special_tokens):
-    tokenizer = HuggingFaceTokenizer(model_path)
+    tokenizer = Tokenizer(model_path).model
     encoded = tokenizer.encode(input,
                                False,
                                add_special_tokens=add_special_tokens)