diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..26d3352 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml diff --git a/.idea/DemoGPT.iml b/.idea/DemoGPT.iml new file mode 100644 index 0000000..8a05c6e --- /dev/null +++ b/.idea/DemoGPT.iml @@ -0,0 +1,12 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..65b1d5d --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,21 @@ + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..dc9ea49 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..24e72a4 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/README.md b/README.md index 0347a8d..9928b54 100644 --- a/README.md +++ b/README.md @@ -202,6 +202,16 @@ If you have cloned the repository and wish to run the source code version, you c streamlit run demogpt/app.py ``` +### Use Local LLM + +First, modify the configuration in model_config.py to ensure that the desired model is correctly configured for use. +To start the LLM server by running the following command: + +```sh +cd server +python llm_api.py +``` + ## To-Do 📝 - [x] Implement new DemoGPT pipeline including plan generation, task creation, code snippet generation, and final code assembly. - [x] Add feature to allow users to select models. diff --git a/demogpt/app.py b/demogpt/app.py index 0a23359..3dbc22e 100644 --- a/demogpt/app.py +++ b/demogpt/app.py @@ -13,6 +13,7 @@ from model import DemoGPT from utils import runStreamlit +from model_config import llm_model_dict # logging.basicConfig(level = logging.DEBUG,format='%(levelname)s-%(message)s') @@ -52,32 +53,38 @@ def initCode(): initCode() -# Text input - -openai_api_key = st.sidebar.text_input( - "OpenAI API Key", - placeholder="sk-...", - value=os.getenv("OPENAI_API_KEY", ""), - type="password", -) -openai_api_base = st.sidebar.text_input( - "Open AI base URL", - placeholder="https://api.openai.com/v1", +llm = st.sidebar.selectbox( + "LLM models", + llm_model_dict.keys(), ) -models = ( - "gpt-3.5-turbo-0613", - "gpt-3.5-turbo-0301", - "gpt-3.5-turbo", - "gpt-3.5-turbo-16k", - "gpt-3.5-turbo-16k-0613", - "gpt-4", - "gpt-4-0314", - "gpt-4-0613", -) +# Text input -model_name = st.sidebar.selectbox("Model", models) +# openai_api_key = st.sidebar.text_input( +# "OpenAI API Key", +# placeholder="sk-...", +# value=os.getenv("OPENAI_API_KEY", ""), +# type="password", +# ) +# +# openai_api_base = st.sidebar.text_input( +# "Open AI base URL", +# placeholder="https://api.openai.com/v1", +# ) + +# models = ( +# "gpt-3.5-turbo-0613", +# "gpt-3.5-turbo-0301", +# "gpt-3.5-turbo", +# "gpt-3.5-turbo-16k", +# "gpt-3.5-turbo-16k-0613", +# "gpt-4", +# "gpt-4-0314", +# "gpt-4-0613", +# ) +# +# model_name = st.sidebar.selectbox("Model", models) empty_idea = st.empty() demo_idea = empty_idea.text_area( @@ -119,13 +126,14 @@ def kill(): if submitted: st.session_state.messages = [] - if not openai_api_key: - st.warning("Please enter your OpenAI API Key!", icon="⚠️") + if not llm: + st.warning("Please choose llm model!", icon="⚠️") else: bar = progressBar(0) st.session_state.container = st.container() - agent = DemoGPT(openai_api_key=openai_api_key, openai_api_base=openai_api_base) - agent.setModel(model_name) + agent = DemoGPT(llm_config=llm_model_dict[llm]) + # agent = DemoGPT(openai_api_key=openai_api_key, openai_api_base=openai_api_base) + # agent.setModel(model_name) kill() code_empty = st.empty() st.session_state.container = st.container() diff --git a/demogpt/model.py b/demogpt/model.py index 9281eac..45a7b52 100644 --- a/demogpt/model.py +++ b/demogpt/model.py @@ -12,18 +12,13 @@ class DemoGPT: def __init__( self, - openai_api_key=os.getenv("OPENAI_API_KEY", ""), - model_name="gpt-3.5-turbo-0613", + llm_config, max_steps=10, - openai_api_base="", ): - assert len( - openai_api_key.strip() - ), "Either give openai_api_key as an argument or put it in the environment variable" - self.model_name = model_name - self.openai_api_key = openai_api_key + self.model_name = llm_config["model_name"] + self.openai_api_base = llm_config["api_base_url"] + self.openai_api_key = llm_config["api_key"] self.max_steps = max_steps # max iteration for refining the model purpose - self.openai_api_base = openai_api_base Chains.setLlm( self.model_name, self.openai_api_key, openai_api_base=self.openai_api_base ) diff --git a/demogpt/model_config.py b/demogpt/model_config.py new file mode 100644 index 0000000..80d81b4 --- /dev/null +++ b/demogpt/model_config.py @@ -0,0 +1,62 @@ +import os +import logging +import torch +# 日志格式 +LOG_FORMAT = "%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s" +logger = logging.getLogger() +logger.setLevel(logging.INFO) +logging.basicConfig(format=LOG_FORMAT) + +llm_model_dict = { + "chatglm-6b": { + "model_name": "chatglm-6b", + "local_model_path": "/opt/ChatGLM-6B/chatglm-6b/", + "api_base_url": "http://localhost:8888/v1", # "name"修改为fastchat服务中的"api_base_url" + "api_key": "EMPTY" + }, + + "chatglm2-6b": { + "model_name": "chatglm2-6b", + "local_model_path": "/opt/ChatGLM2-6B/chatglm2-6b/", + "api_base_url": "http://localhost:8888/v1", # "name"修改为fastchat服务中的"api_base_url" + "api_key": "EMPTY" + }, + + + "vicuna-13b-hf": { + "local_model_path": "vicuna-13b-hf", + "api_base_url": "http://localhost:8888/v1", # "name"修改为fastchat服务中的"api_base_url" + "api_key": "EMPTY" + }, + + "gpt-3.5-turbo": { + "local_model_path": "gpt-3.5-turbo", + "api_base_url": "https://api.openai.com/v1", + "api_key": os.environ.get("OPENAI_API_KEY") + }, + + "baichuan-7b": { + "model_name": "baichuan-7b", + "local_model_path": "/opt/baichuan-7B", + "api_base_url": "http://localhost:8888/v1", # "name"修改为fastchat服务中的"api_base_url" + "api_key": "EMPTY" + }, + "Baichuan-13b-Chat": { + "model_name": "baichuan-13b", + "local_model_path": "baichuan-inc/Baichuan-13b-Chat", + "api_base_url": "http://localhost:8888/v1", # "name"修改为fastchat服务中的"api_base_url" + "api_key": "EMPTY" + }, + +} + +# LLM 名称 +LLM_MODEL = "chatglm-6b" + +# LLM 运行设备 +LLM_DEVICE = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu" + +# 日志存储路径 +LOG_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), "logs") +if not os.path.exists(LOG_PATH): + os.mkdir(LOG_PATH) diff --git a/demogpt/server/__init__.py b/demogpt/server/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/demogpt/server/llm_api.py b/demogpt/server/llm_api.py new file mode 100644 index 0000000..f18a24c --- /dev/null +++ b/demogpt/server/llm_api.py @@ -0,0 +1,235 @@ +from multiprocessing import Process, Queue +import sys +import os + +sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +from demogpt.model_config import llm_model_dict, LLM_MODEL, LLM_DEVICE, LOG_PATH, logger + + +host_ip = "0.0.0.0" +controller_port = 20001 +model_worker_port = 20002 +openai_api_port = 8888 +base_url = "http://127.0.0.1:{}" +queue = Queue() + + +def set_httpx_timeout(timeout=60.0): + import httpx + httpx._config.DEFAULT_TIMEOUT_CONFIG.connect = timeout + httpx._config.DEFAULT_TIMEOUT_CONFIG.read = timeout + httpx._config.DEFAULT_TIMEOUT_CONFIG.write = timeout + + +def create_controller_app( + dispatch_method="shortest_queue", +): + import fastchat.constants + fastchat.constants.LOGDIR = LOG_PATH + from fastchat.serve.controller import app, Controller + + controller = Controller(dispatch_method) + sys.modules["fastchat.serve.controller"].controller = controller + + return app + + +def create_model_worker_app( + model_path=llm_model_dict[LLM_MODEL].get("local_model_path"), + model_names=[LLM_MODEL], + device=LLM_DEVICE, + load_8bit=False, + gptq_ckpt=None, + gptq_wbits=16, + gptq_groupsize=-1, + gptq_act_order=None, + gpus="0,1", + num_gpus=2, + max_gpu_memory="15GiB", + cpu_offloading=None, + worker_address=base_url.format(model_worker_port), + controller_address=base_url.format(controller_port), + limit_worker_concurrency=5, + stream_interval=2, + no_register=False, +): + import fastchat.constants + fastchat.constants.LOGDIR = LOG_PATH + from fastchat.serve.model_worker import app, GptqConfig, ModelWorker, worker_id + from fastchat.serve import model_worker + import argparse + + parser = argparse.ArgumentParser() + args = parser.parse_args() + args.model_path = model_path + args.model_names = model_names + args.device = device + args.load_8bit = load_8bit + args.gptq_ckpt = gptq_ckpt + args.gptq_wbits = gptq_wbits + args.gptq_groupsize = gptq_groupsize + args.gptq_act_order = gptq_act_order + args.gpus = gpus + args.num_gpus = num_gpus + args.max_gpu_memory = max_gpu_memory + args.cpu_offloading = cpu_offloading + args.worker_address = worker_address + args.controller_address = controller_address + args.limit_worker_concurrency = limit_worker_concurrency + args.stream_interval = stream_interval + args.no_register = no_register + + if args.gpus: + if len(args.gpus.split(",")) < args.num_gpus: + raise ValueError( + f"Larger --num-gpus ({args.num_gpus}) than --gpus {args.gpus}!" + ) + os.environ["CUDA_VISIBLE_DEVICES"] = args.gpus + + if gpus and num_gpus is None: + num_gpus = len(gpus.split(',')) + args.num_gpus = num_gpus + + gptq_config = GptqConfig( + ckpt=gptq_ckpt or model_path, + wbits=args.gptq_wbits, + groupsize=args.gptq_groupsize, + act_order=args.gptq_act_order, + ) + # torch.multiprocessing.set_start_method('spawn') + worker = ModelWorker( + controller_addr=args.controller_address, + worker_addr=args.worker_address, + worker_id=worker_id, + model_path=args.model_path, + model_names=args.model_names, + limit_worker_concurrency=args.limit_worker_concurrency, + no_register=args.no_register, + device=args.device, + num_gpus=args.num_gpus, + max_gpu_memory=args.max_gpu_memory, + load_8bit=args.load_8bit, + cpu_offloading=args.cpu_offloading, + gptq_config=gptq_config, + stream_interval=args.stream_interval, + ) + + sys.modules["fastchat.serve.model_worker"].worker = worker + sys.modules["fastchat.serve.model_worker"].args = args + sys.modules["fastchat.serve.model_worker"].gptq_config = gptq_config + + return app + + +def create_openai_api_app( + host=host_ip, + port=openai_api_port, + controller_address=base_url.format(controller_port), + api_keys=[], +): + import fastchat.constants + fastchat.constants.LOGDIR = LOG_PATH + from fastchat.serve.openai_api_server import app, CORSMiddleware, app_settings + + app.add_middleware( + CORSMiddleware, + allow_credentials=True, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], + ) + + app_settings.controller_address = controller_address + app_settings.api_keys = api_keys + + return app + + +def run_controller(q): + import uvicorn + app = create_controller_app() + + @app.on_event("startup") + async def on_startup(): + set_httpx_timeout() + q.put(1) + + uvicorn.run(app, host=host_ip, port=controller_port) + + +def run_model_worker(q, *args, **kwargs): + import uvicorn + app = create_model_worker_app(*args, **kwargs) + + @app.on_event("startup") + async def on_startup(): + set_httpx_timeout() + while True: + no = q.get() + if no != 1: + q.put(no) + else: + break + q.put(2) + + uvicorn.run(app, host=host_ip, port=model_worker_port) + + +def run_openai_api(q): + import uvicorn + app = create_openai_api_app() + + @app.on_event("startup") + async def on_startup(): + set_httpx_timeout() + while True: + no = q.get() + if no != 2: + q.put(no) + else: + break + q.put(3) + + uvicorn.run(app, host=host_ip, port=openai_api_port) + + +if __name__ == "__main__": + logger.info(llm_model_dict[LLM_MODEL]) + model_path = llm_model_dict[LLM_MODEL]["local_model_path"] + + logger.info(f"如需查看 llm_api 日志，请前往 {LOG_PATH}") + + if not model_path: + logger.error("local_model_path 不能为空") + else: + controller_process = Process( + target=run_controller, + name=f"controller({os.getpid()})", + args=(queue,), + daemon=True, + ) + controller_process.start() + + # cuda 没办法用在fork的多进程中 + # model_worker_process = Process( + # target=run_model_worker, + # name=f"model_worker({os.getpid()})", + # args=(queue,), + # # kwargs={"load_8bit": True}, + # daemon=True, + # ) + # model_worker_process.start() + + openai_api_process = Process( + target=run_openai_api, + name=f"openai_api({os.getpid()})", + args=(queue,), + daemon=True, + ) + openai_api_process.start() + + run_model_worker(queue) + + controller_process.join() + # model_worker_process.join() + openai_api_process.join()