update config.yaml

Author: shell-nlp

gpt_server/model_worker/base/model_worker_base.py

@@ -176,6 +176,10 @@ def run(cls):
             "--model_names", type=lambda s: s.split(","), default="model_names"
         )
         parser.add_argument("--lora", type=str, default=None)
+        parser.add_argument("--host", type=str, default="localhost")
+        parser.add_argument(
+            "--controller_address", type=str, default="http://localhost:21001"
+        )
 
         args = parser.parse_args()
         os.environ["num_gpus"] = str(args.num_gpus)
@@ -189,8 +193,9 @@ def run(cls):
             os.environ["backend"] = "lmdeploy-turbomind"
         if args.lora:
             os.environ["lora"] = args.lora
+        host = args.host
+        controller_address = args.controller_address
 
-        host = "localhost"
         port = get_free_tcp_port()
         worker_addr = f"http://{host}:{port}"
 
@@ -203,6 +208,7 @@ async def startup():
                 model_path=args.model_name_or_path,
                 model_names=args.model_names,
                 conv_template="chatglm3",  # TODO 默认是chatglm3用于统一处理
+                controller_addr=controller_address,
             )
 
         uvicorn.run(app, host=host, port=port)

gpt_server/script/config.yaml

@@ -8,11 +8,15 @@ serve_args:
 
 # controller
 controller_args:
-  host: localhost
+  host: 0.0.0.0
   port: 21001
   dispatch_method: shortest_queue # lottery shortest_queue
 
 # model worker
+model_worker_args:
+  host: 0.0.0.0
+  controller_address: http://localhost:21001
+
 models:
   - glm-4v:  #自定义的模型名称
       alias: null # 别名     例如  gpt4,gpt3
@@ -52,7 +56,7 @@ models:
 
   - qwen:  #自定义的模型名称
       alias: gpt-4,gpt-3.5-turbo,gpt-3.5-turbo-16k # 别名     例如  gpt4,gpt3
-      enable: true  # false true
+      enable: false  # false true
       model_name_or_path: /home/dev/model/qwen/Qwen1___5-14B-Chat/ 
       model_type: qwen  # qwen  chatglm3 yi internlm
       work_mode: vllm  # vllm hf lmdeploy-turbomind  lmdeploy-pytorch
@@ -66,7 +70,7 @@ models:
       # - gpus:
       #   - 3
   - qwen-72b:  #自定义的模型名称
-      alias: null # 别名     例如  gpt4,gpt3
+      alias: qwen # 别名     例如  gpt4,gpt3
       enable: true  # false true
       model_name_or_path: /home/dev/model/qwen/Qwen2-72B-Instruct-AWQ/
       model_type: qwen  # qwen  chatglm3 yi internlm

gpt_server/utils.py

@@ -56,6 +56,14 @@ def start_api_server(config: dict):
 
 def start_model_worker(config: dict):
     process = []
+    try:
+        host = config["model_worker_args"]["host"]
+        controller_address = config["model_worker_args"]["controller_address"]
+    except KeyError as e:
+        error_msg = f"请参照 https://github.com/shell-nlp/gpt_server/blob/main/gpt_server/script/config.yaml 设置正确的 model_worker_args"
+        logger.error(error_msg)
+        raise KeyError(error_msg)
+
     for model_config_ in config["models"]:
         for model_name, model_config in model_config_.items():
             # 启用的模型
@@ -110,6 +118,8 @@ def start_model_worker(config: dict):
                         + f" --model_name_or_path {model_name_or_path}"
                         + f" --model_names {model_names}"
                         + f" --backend {backend}"
+                        + f" --host {host}"
+                        + f" --controller_address {controller_address}"
                     )
                     if lora:
                         cmd += f" --lora '{json.dumps(lora)}'"