合并后的提交信息

Author: shell-nlp

gpt_server/model_backend/hf_backend.py

@@ -55,6 +55,7 @@ def __init__(self, tokenizer: PreTrainedTokenizer, model: torch.nn.Module) -> No
                 self.model.load_adapter(model_id=lora_path, adapter_name=lora_name)
 
     async def stream_chat(self, params: Dict[str, Any]):
+        # params 已不需要传入 prompt
         messages = params["messages"]
         chat_template = params.get("chat_template", None)
         tools = params.get("tools", None)

gpt_server/model_worker/auto.py

@@ -0,0 +1,83 @@
+import asyncio
+import json
+from typing import List
+from fastchat.constants import ErrorCode, SERVER_ERROR_MSG
+from loguru import logger
+import torch
+import traceback
+from gpt_server.model_worker.base.model_worker_base import ModelWorkerBase
+from gpt_server.model_handler.prompts import MODELS
+from gpt_server.model_handler.tool_parser import tool_parser, ToolParserManager
+from gpt_server.model_handler.chat_template.get_chat_template import get_chat_template
+
+
+class AutoWorker(ModelWorkerBase):
+    def __init__(
+        self,
+        controller_addr: str,
+        worker_addr: str,
+        worker_id: str,
+        model_path: str,
+        model_names: List[str],
+        limit_worker_concurrency: int,
+        conv_template: str = None,  # type: ignore
+    ):
+        super().__init__(
+            controller_addr,
+            worker_addr,
+            worker_id,
+            model_path,
+            model_names,
+            limit_worker_concurrency,
+            conv_template,
+            model_type="AutoModelForCausalLM",
+        )
+
+        self.stop_words_ids = []
+
+        self.stop = [
+            self.tokenizer.decode(skip_word) for skip_word in self.stop_words_ids
+        ]
+        logger.warning(f"{model_names[0]} 停用词: {self.stop}")
+
+        # from https://github.com/xorbitsai/inference/blob/c70ea74fa820a613f8d577047ef1818da20a96b3/xinference/model/llm/llm_family_modelscope.json
+        self.tool_parser = ToolParserManager.module_dict["qwen2_5"](
+            tokenizer=self.tokenizer
+        )
+
+    async def generate_stream_gate(self, params):
+        self.call_ct += 1
+        try:
+            tools = params.get("tools", None)
+            # ---------------添加额外的参数------------------------
+            params["stop"].extend(self.stop)
+            params["stop_words_ids"] = self.stop_words_ids
+            # ---------------添加额外的参数------------------------
+            full_text = ""
+            ret = {}
+            async for ret in self.backend.stream_chat(params=params):
+                full_text += ret.get("text", "")
+                yield json.dumps(ret).encode() + b"\0"
+            # ------ add tool_calls ------
+            yield tool_parser(
+                full_text=full_text, tool_parser=self.tool_parser, tools=tools, ret=ret
+            )
+            # ------ add tool_calls ------
+        except torch.cuda.OutOfMemoryError as e:
+            ret = {
+                "text": f"{SERVER_ERROR_MSG}\n\n({e})",
+                "error_code": ErrorCode.CUDA_OUT_OF_MEMORY,
+            }
+            yield json.dumps(ret).encode() + b"\0"
+        except (ValueError, RuntimeError) as e:
+            traceback.print_exc()
+            logger.info(e)
+            ret = {
+                "text": f"{SERVER_ERROR_MSG}\n\n({e})",
+                "error_code": ErrorCode.INTERNAL_ERROR,
+            }
+            yield json.dumps(ret).encode() + b"\0"
+
+
+if __name__ == "__main__":
+    AutoWorker.run()

gpt_server/model_worker/baichuan.py

@@ -6,57 +6,6 @@
 from gpt_server.model_worker.base.model_worker_base import ModelWorkerBase
 
 
-def build_chat_input(tokenizer, messages: List[dict], max_new_tokens: int = 0):
-    user_token_id = 195
-    assistant_token_id = 196
-
-    def _parse_messages(messages, split_role="user"):
-        system, rounds = "", []
-        round = []
-        for i, message in enumerate(messages):
-            if message["role"] == "system":
-                assert i == 0
-                system = message["content"]
-                continue
-            if message["role"] == split_role and round:
-                rounds.append(round)
-                round = []
-            round.append(message)
-        if round:
-            rounds.append(round)
-        return system, rounds
-
-    max_new_tokens = max_new_tokens or 2048
-    max_input_tokens = 4096 - max_new_tokens
-    system, rounds = _parse_messages(messages, split_role="user")
-    system_tokens = tokenizer.encode(system)
-    max_history_tokens = max_input_tokens - len(system_tokens)
-
-    history_tokens = []
-    for round in rounds[::-1]:
-        round_tokens = []
-        for message in round:
-            if message["role"] == "user":
-                round_tokens.append(user_token_id)
-            else:
-                round_tokens.append(assistant_token_id)
-            round_tokens.extend(tokenizer.encode(message["content"]))
-        if (
-            len(history_tokens) == 0
-            or len(history_tokens) + len(round_tokens) <= max_history_tokens
-        ):
-            history_tokens = round_tokens + history_tokens  # concat left
-            if len(history_tokens) < max_history_tokens:
-                continue
-        break
-
-    input_tokens = system_tokens + history_tokens
-    if messages[-1]["role"] != "assistant":
-        input_tokens.append(assistant_token_id)
-    input_tokens = input_tokens[-max_input_tokens:]  # truncate left
-    return torch.LongTensor([input_tokens])
-
-
 class BaiChuanWorker(ModelWorkerBase):
     def __init__(
         self,
@@ -78,9 +27,7 @@ def __init__(
             conv_template,
             model_type="AutoModelForCausalLM",
         )
-        self.stop_words_ids = [
-            2,  # </s>
-        ]
+        self.stop_words_ids = []
         self.stop = [
             self.tokenizer.decode(skip_word) for skip_word in self.stop_words_ids
         ]
@@ -89,29 +36,11 @@ def __init__(
     async def generate_stream_gate(self, params):
         self.call_ct += 1
         try:
-            messages = params["messages"]
-            if isinstance(messages, list):
-                task = "chat"
-            elif isinstance(messages, str):
-                task = "completion"
-            if task == "chat":
-                input_ids = build_chat_input(
-                    tokenizer=self.tokenizer, messages=messages
-                )
-                text = self.tokenizer.decode(input_ids.tolist()[0])
-            elif task == "completion":
-                text = messages
-                input_ids = self.tokenizer([text], return_tensors="pt").input_ids
-
-            params["messages"] = messages
-            params["prompt"] = text
             params["stop"].extend(self.stop)
             params["stop_words_ids"] = self.stop_words_ids
-            params["input_ids"] = input_ids
 
             async for ret in self.backend.stream_chat(params=params):
                 response = ret["text"]
-
                 yield json.dumps(ret).encode() + b"\0"
 
         except torch.cuda.OutOfMemoryError as e:

gpt_server/model_worker/base/model_worker_base.py

@@ -137,19 +137,20 @@ def __init__(
 
     def preprocess_params(self, params: dict) -> dict:
         """预处理 params"""
-        messages = params["messages"]
+        # ---------- 添加 chat_template 信息 ----------
         params["chat_template"] = self.chat_template
+        # ---------- 添加多模态信息 ----------
         if self.vision_config:
             params["multimodal"] = True
             params["chat_template"] = self.vl_chat_template
+        # ---------- 如果传入的是 str 则修改为messages ----------
+        messages = params["messages"]
         if isinstance(messages, str):
             messages = [{"role": "user", "content": messages}]
             params["messages"] = messages
-        # 1. 处理 工具，支持 tool_choice 的控制
+        # ---------- 处理 工具，支持 tool_choice 的控制 ----------
         tool_choice = params.get("tool_choice", "none")
         tools = params.get("tools", None)
-        if self.chat_template:
-            params["chat_template"] = self.chat_template
         params["extra_prompt"] = ""
         if tools:
             if tool_choice == "none":
@@ -404,6 +405,8 @@ async def api_generate_stream(request: Request):
     params["request"] = request
     params.pop("prompt")
     logger.debug(f"params {params}")
+    # 对 params 进行预处理
+    params = worker.preprocess_params(params)
     generator = worker.generate_stream_gate(params)
     background_tasks = create_background_tasks(request_id)
     return StreamingResponse(generator, background=background_tasks)
@@ -450,6 +453,8 @@ async def api_generate(request: Request):
     params["request"] = request
     params.pop("prompt")
     logger.debug(f"params {params}")
+    # 对 params 进行预处理
+    params = worker.preprocess_params(params)
     output = await worker.generate_gate(params)
     release_worker_semaphore()
 

gpt_server/model_worker/chatglm.py

@@ -44,46 +44,20 @@ def __init__(
             model_type="AutoModel",
             multimodal=False,
         )
-        self.chat_template = MODELS.module_dict["glm4"]()
         self.tool_parser = ToolParserManager.module_dict["glm"](
             tokenizer=self.tokenizer
         )
         self.stop_words_ids = []
-
         self.stop = ["Observation:"]
         logger.warning(f"{model_names[0]} 停用词: {self.stop}")
 
     async def generate_stream_gate(self, params):
         self.call_ct += 1
         try:
-            messages = params.get("messages", [])
             tools = params.get("tools", None)
-            tool_choice = params.get("tool_choice", "none")
-            if tool_choice == "none":
-                tools = None
-            elif tool_choice == "auto" or tool_choice == "required":
-                pass
-            elif isinstance(tool_choice, dict):
-                tools = pop_matching_tool(tools=tools, tool_choice=tool_choice)
-            if not self.vision_config:
-                if isinstance(messages, list):
-                    text = await asyncio.to_thread(
-                        self.chat_template.messages2prompt, messages, True, tools
-                    )
-                elif isinstance(messages, str):
-                    text = messages
-                    # input_ids = self.tokenizer([text], return_tensors="pt").input_ids
 
-                # text = self.tokenizer.decode(input_ids.tolist()[0])
-                params["prompt"] = text
-                # params["input_ids"] = input_ids
-            else:  # 多模态模型
-                params["multimodal"] = True
-            # ---------------添加额外的参数------------------------
-            params["messages"] = messages
             params["stop"].extend(self.stop)
             params["stop_words_ids"] = self.stop_words_ids
-            # ---------------添加额外的参数------------------------
             full_text = ""
             ret = {}
             async for ret in self.backend.stream_chat(params=params):

gpt_server/model_worker/deepseek.py

@@ -28,10 +28,7 @@ def __init__(
             model_type="AutoModelForCausalLM",
         )
 
-        self.stop_words_ids = [
-            # 32013,  # bos  <｜begin▁of▁sentence｜>
-            # 32021,  # eos  <|EOT|>
-        ]
+        self.stop_words_ids = []
 
         self.stop = [
             self.tokenizer.decode(skip_word) for skip_word in self.stop_words_ids
@@ -41,19 +38,6 @@ def __init__(
     async def generate_stream_gate(self, params):
         self.call_ct += 1
         try:
-            messages = params["messages"]
-            if not self.vision_config:
-                if isinstance(messages, list):
-                    text = self.tokenizer.apply_chat_template(
-                        conversation=messages,
-                        tokenize=False,
-                        add_generation_prompt=True,
-                    )
-                elif isinstance(messages, str):
-                    text = messages
-                params["prompt"] = text
-            # ---------------添加额外的参数------------------------
-            params["messages"] = messages
             params["stop"].extend(self.stop)
             params["stop_words_ids"] = self.stop_words_ids
             # ---------------添加额外的参数------------------------

gpt_server/model_worker/gemma.py

@@ -6,6 +6,7 @@
 from gpt_server.model_worker.base.model_worker_base import ModelWorkerBase
 import traceback
 
+
 class GemmaWorker(ModelWorkerBase):
     def __init__(
         self,
@@ -27,7 +28,7 @@ def __init__(
             conv_template,
             model_type="AutoModelForCausalLM",
         )
-        self.stop_words_ids = [1, 106]
+        self.stop_words_ids = []
         self.stop = [
             self.tokenizer.decode(skip_word) for skip_word in self.stop_words_ids
         ]
@@ -36,28 +37,11 @@ def __init__(
     async def generate_stream_gate(self, params):
         self.call_ct += 1
         try:
-            messages = params["messages"]
-            if isinstance(messages, list):
-                task = "chat"
-            elif isinstance(messages, str):
-                task = "completion"
-            if task == "chat":
-                text = self.tokenizer.apply_chat_template(
-                    conversation=messages,
-                    tokenize=True,
-                    add_generation_prompt=True,
-                )
-            elif task == "completion":
-                text = messages
-
-            params["messages"] = messages
-            params["prompt"] = text
             params["stop"].extend(self.stop)
             params["stop_words_ids"] = self.stop_words_ids
 
             async for ret in self.backend.stream_chat(params=params):
                 response = ret["text"]
-
                 yield json.dumps(ret).encode() + b"\0"
 
         except torch.cuda.OutOfMemoryError as e: