From 8642f385e4233a1aaa99f0b800a436a853494505 Mon Sep 17 00:00:00 2001 From: ZeyuTeng96 <96521059+ZeyuTeng96@users.noreply.github.com> Date: Tue, 31 Oct 2023 15:55:44 +0800 Subject: [PATCH 1/6] add chatglm3 conv template support in conversation.py Hi there, by checking following tokenizer script, I think the true chatglm3 conv templare is like: "<|system|>\nYou are ChatGLM3, a large language model trained by Zhipu.AI. Follow the user's instructions carefully. Respond using markdown.<|user|>\nHello!<|assistant|>\nHi!<|user|>\nHow are you?<|assistant|>" please see following code: https://huggingface.co/THUDM/chatglm3-6b/blob/fc3235f807ef5527af598c05f04f2ffd17f48bab/tokenization_chatglm.py#L179 https://huggingface.co/THUDM/chatglm3-6b/blob/fc3235f807ef5527af598c05f04f2ffd17f48bab/tokenization_chatglm.py#L194 --- fastchat/conversation.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/fastchat/conversation.py b/fastchat/conversation.py index a8bdb1cb6..da1a934b4 100644 --- a/fastchat/conversation.py +++ b/fastchat/conversation.py @@ -28,6 +28,7 @@ class SeparatorStyle(IntEnum): PHOENIX = auto() ROBIN = auto() FALCON_CHAT = auto() + CHATGLM3 = auto() @dataclasses.dataclass @@ -163,6 +164,14 @@ def get_prompt(self) -> str: else: ret += role + "\n" return ret + elif self.sep_style == SeparatorStyle.CHATGLM3: + ret = "" if system_prompt == "" else system_prompt + for role, message in self.messages: + if message: + ret += role + "\n" + message + else: + ret += role + return ret elif self.sep_style == SeparatorStyle.CHATINTERN: # source: https://huggingface.co/internlm/internlm-chat-7b-8k/blob/bd546fa984b4b0b86958f56bf37f94aa75ab8831/modeling_internlm.py#L771 seps = [self.sep, self.sep2] @@ -448,6 +457,23 @@ def get_conv_template(name: str) -> Conversation: ) ) +# ChatGLM3 default template +register_conv_template( + Conversation( + name="chatglm3", + system_template="<|system|>\n{system_message}", + system_message="You are ChatGLM3, a large language model trained by Zhipu.AI. Follow the user's instructions carefully. Respond using markdown.", + roles=("<|user|>", "<|assistant|>"), + sep_style=SeparatorStyle.CHATGLM3, + sep="", + stop_token_ids=[ + 64795, + 64797, + 2, + ], # "<|user|>", "<|observation|>", "" + ) +) + # Dolly V2 default template register_conv_template( Conversation( From 2183bc4ef91529491e4cda42c5d1b9038d7b5892 Mon Sep 17 00:00:00 2001 From: ZeyuTeng96 <96521059+ZeyuTeng96@users.noreply.github.com> Date: Fri, 10 Nov 2023 11:10:49 +0800 Subject: [PATCH 2/6] Update conversation.py --- fastchat/conversation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fastchat/conversation.py b/fastchat/conversation.py index da1a934b4..a5fe629b8 100644 --- a/fastchat/conversation.py +++ b/fastchat/conversation.py @@ -168,7 +168,7 @@ def get_prompt(self) -> str: ret = "" if system_prompt == "" else system_prompt for role, message in self.messages: if message: - ret += role + "\n" + message + ret += role + "\n" + " " + message else: ret += role return ret @@ -461,7 +461,7 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="chatglm3", - system_template="<|system|>\n{system_message}", + system_template="<|system|>\n {system_message}", system_message="You are ChatGLM3, a large language model trained by Zhipu.AI. Follow the user's instructions carefully. Respond using markdown.", roles=("<|user|>", "<|assistant|>"), sep_style=SeparatorStyle.CHATGLM3, From c92675f461a1c66bd4e49a218a6300a4d9994c3a Mon Sep 17 00:00:00 2001 From: ZeyuTeng96 <96521059+ZeyuTeng96@users.noreply.github.com> Date: Fri, 10 Nov 2023 11:50:39 +0800 Subject: [PATCH 3/6] Update model_adapter.py to support chatglm3 --- fastchat/model/model_adapter.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/fastchat/model/model_adapter.py b/fastchat/model/model_adapter.py index ca03e69e4..16ead9658 100644 --- a/fastchat/model/model_adapter.py +++ b/fastchat/model/model_adapter.py @@ -754,9 +754,15 @@ def match(self, model_path: str): def load_model(self, model_path: str, from_pretrained_kwargs: dict): revision = from_pretrained_kwargs.get("revision", "main") - tokenizer = AutoTokenizer.from_pretrained( - model_path, trust_remote_code=True, revision=revision - ) + if "chatglm3" in model_path.lower(): + # source: https://huggingface.co/THUDM/chatglm3-6b/commit/e46a14881eae613281abbd266ee918e93a56018f + tokenizer = AutoTokenizer.from_pretrained( + model_path, encode_special_tokens=True, trust_remote_code=True, revision=revision + ) + else: + tokenizer = AutoTokenizer.from_pretrained( + model_path, trust_remote_code=True, revision=revision + ) model = AutoModel.from_pretrained( model_path, trust_remote_code=True, **from_pretrained_kwargs ) @@ -766,6 +772,8 @@ def get_default_conv_template(self, model_path: str) -> Conversation: model_path = model_path.lower() if "chatglm2" in model_path.lower(): return get_conv_template("chatglm2") + if "chatglm3" in model_path.lower(): + return get_conv_template("chatglm3") return get_conv_template("chatglm") From dcd532439d24b7ba3752eb41ccbffe53171f5687 Mon Sep 17 00:00:00 2001 From: ZeyuTeng96 <96521059+ZeyuTeng96@users.noreply.github.com> Date: Fri, 10 Nov 2023 11:55:59 +0800 Subject: [PATCH 4/6] Update model_adapter.py --- fastchat/model/model_adapter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fastchat/model/model_adapter.py b/fastchat/model/model_adapter.py index 16ead9658..609268a00 100644 --- a/fastchat/model/model_adapter.py +++ b/fastchat/model/model_adapter.py @@ -755,7 +755,6 @@ def match(self, model_path: str): def load_model(self, model_path: str, from_pretrained_kwargs: dict): revision = from_pretrained_kwargs.get("revision", "main") if "chatglm3" in model_path.lower(): - # source: https://huggingface.co/THUDM/chatglm3-6b/commit/e46a14881eae613281abbd266ee918e93a56018f tokenizer = AutoTokenizer.from_pretrained( model_path, encode_special_tokens=True, trust_remote_code=True, revision=revision ) From efd1501f2c7063429efc753945567c11477737f8 Mon Sep 17 00:00:00 2001 From: ZeyuTeng96 <96521059+ZeyuTeng96@users.noreply.github.com> Date: Fri, 10 Nov 2023 12:21:28 +0800 Subject: [PATCH 5/6] Update model_adapter.py --- fastchat/model/model_adapter.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fastchat/model/model_adapter.py b/fastchat/model/model_adapter.py index 609268a00..9a2424d57 100644 --- a/fastchat/model/model_adapter.py +++ b/fastchat/model/model_adapter.py @@ -756,7 +756,10 @@ def load_model(self, model_path: str, from_pretrained_kwargs: dict): revision = from_pretrained_kwargs.get("revision", "main") if "chatglm3" in model_path.lower(): tokenizer = AutoTokenizer.from_pretrained( - model_path, encode_special_tokens=True, trust_remote_code=True, revision=revision + model_path, + encode_special_tokens=True, + trust_remote_code=True, + revision=revision, ) else: tokenizer = AutoTokenizer.from_pretrained( From 99087e7719a09e5344115be077f6166e5b88035f Mon Sep 17 00:00:00 2001 From: ZeyuTeng96 <96521059+ZeyuTeng96@users.noreply.github.com> Date: Fri, 10 Nov 2023 14:34:41 +0800 Subject: [PATCH 6/6] Update conversation.py --- fastchat/conversation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fastchat/conversation.py b/fastchat/conversation.py index b25e753bc..6067710e8 100644 --- a/fastchat/conversation.py +++ b/fastchat/conversation.py @@ -165,7 +165,9 @@ def get_prompt(self) -> str: ret += role + "\n" return ret elif self.sep_style == SeparatorStyle.CHATGLM3: - ret = "" if system_prompt == "" else system_prompt + ret = "" + if self.system_message: + ret += system_prompt for role, message in self.messages: if message: ret += role + "\n" + " " + message @@ -462,10 +464,8 @@ def get_conv_template(name: str) -> Conversation: Conversation( name="chatglm3", system_template="<|system|>\n {system_message}", - system_message="You are ChatGLM3, a large language model trained by Zhipu.AI. Follow the user's instructions carefully. Respond using markdown.", roles=("<|user|>", "<|assistant|>"), sep_style=SeparatorStyle.CHATGLM3, - sep="", stop_token_ids=[ 64795, 64797,