From e198cd34aea1cf4864ce46330ba558ea997f5164 Mon Sep 17 00:00:00 2001 From: luchun <71970539+zhanghy-sketchzh@users.noreply.github.com> Date: Thu, 16 Nov 2023 22:32:14 +0800 Subject: [PATCH 1/3] Update chatgpt.py --- pilot/model/proxy/llms/chatgpt.py | 129 +++++++++++++++++++++++++----- 1 file changed, 108 insertions(+), 21 deletions(-) diff --git a/pilot/model/proxy/llms/chatgpt.py b/pilot/model/proxy/llms/chatgpt.py index ab95e58f6..37bafe193 100644 --- a/pilot/model/proxy/llms/chatgpt.py +++ b/pilot/model/proxy/llms/chatgpt.py @@ -4,7 +4,7 @@ import os from typing import List import logging - +import importlib.metadata as metadata from pilot.model.proxy.llms.proxy_model import ProxyModel from pilot.model.parameter import ProxyModelParameters from pilot.scene.base_message import ModelMessage, ModelMessageRoleType @@ -56,14 +56,52 @@ def _initialize_openai(params: ProxyModelParameters): return openai_params +def _initialize_openai_v1(params: ProxyModelParameters): + try: + from openai import OpenAI + except ImportError as exc: + raise ValueError( + "Could not import python package: openai " + "Please install openai by command `pip install openai" + ) + + api_type = params.proxy_api_type or os.getenv("OPENAI_API_TYPE", "open_ai") + + base_url = params.proxy_api_base or os.getenv( + "OPENAI_API_TYPE", + os.getenv("AZURE_OPENAI_ENDPOINT") if api_type == "azure" else None, + ) + api_key = params.proxy_api_key or os.getenv( + "OPENAI_API_KEY", + os.getenv("AZURE_OPENAI_KEY") if api_type == "azure" else None, + ) + api_version = params.proxy_api_version or os.getenv("OPENAI_API_VERSION") + + if not base_url and params.proxy_server_url: + # Adapt previous proxy_server_url configuration + base_url = params.proxy_server_url.split("/chat/completions")[0] + + if params.http_proxy: + openai.proxies = params.http_proxy + openai_params = { + "api_key": api_key, + "base_url": base_url, + "proxies": params.http_proxy, + } + + return openai_params,api_type,api_version + + + + + def _build_request(model: ProxyModel, params): history = [] model_params = model.get_params() logger.info(f"Model: {model}, model_params: {model_params}") - - openai_params = _initialize_openai(model_params) + messages: List[ModelMessage] = params["messages"] # Add history conversation @@ -95,13 +133,19 @@ def _build_request(model: ProxyModel, params): } proxyllm_backend = model_params.proxyllm_backend - if openai_params["api_type"] == "azure": - # engine = "deployment_name". - proxyllm_backend = proxyllm_backend or "gpt-35-turbo" - payloads["engine"] = proxyllm_backend - else: + if metadata.version("openai") >= "1.0.0": + openai_params,api_type,api_version = _initialize_openai_v1(model_params) proxyllm_backend = proxyllm_backend or "gpt-3.5-turbo" payloads["model"] = proxyllm_backend + else: + openai_params = _initialize_openai(model_params) + if openai_params["api_type"] == "azure": + # engine = "deployment_name". + proxyllm_backend = proxyllm_backend or "gpt-35-turbo" + payloads["engine"] = proxyllm_backend + else: + proxyllm_backend = proxyllm_backend or "gpt-3.5-turbo" + payloads["model"] = proxyllm_backend logger.info( f"Send request to real model {proxyllm_backend}, openai_params: {openai_params}" @@ -112,28 +156,71 @@ def _build_request(model: ProxyModel, params): def chatgpt_generate_stream( model: ProxyModel, tokenizer, params, device, context_len=2048 ): - import openai + if metadata.version("openai") >= "1.0.0": + model_params = model.get_params() + openai_params,api_type,api_version = _initialize_openai_v1(model_params) + history, payloads = _build_request(model, params) + if api_type == "azure": + from openai import AzureOpenAI + client = AzureOpenAI( + api_key = openai_params["api_key"], + api_version = api_version, + azure_endpoint = openai_params["base_url"] # Your Azure OpenAI resource's endpoint value. + ) + else: + from openai import OpenAI + client = OpenAI(**openai_params) + print("openai_params",openai_params) + print("payloads",payloads) + res = client.chat.completions.create(messages=history, **payloads) + print(res) + text = "" + for r in res: + if r.choices[0].delta.content is not None: + content = r.choices[0].delta.content + text += content + yield text - history, payloads = _build_request(model, params) + else: + + import openai + history, payloads = _build_request(model, params) - res = openai.ChatCompletion.create(messages=history, **payloads) + res = openai.ChatCompletion.create(messages=history, **payloads) - text = "" - for r in res: - if r["choices"][0]["delta"].get("content") is not None: - content = r["choices"][0]["delta"]["content"] - text += content - yield text + text = "" + print("res",res) + for r in res: + if r["choices"][0]["delta"].get("content") is not None: + content = r["choices"][0]["delta"]["content"] + text += content + yield text async def async_chatgpt_generate_stream( model: ProxyModel, tokenizer, params, device, context_len=2048 ): - import openai - - history, payloads = _build_request(model, params) + if metadata.version("openai") >= "1.0.0": + model_params = model.get_params() + openai_params,api_type,api_version = _initialize_openai_v1(model_params) + history, payloads = _build_request(model, params) + if api_type == "azure": + from openai import AsyncAzureOpenAI + client = AsyncAzureOpenAI( + api_key = openai_params["api_key"], + end_point = openai_params["base_url"], + api_version = api_version, + azure_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT") # Your Azure OpenAI resource's endpoint value. + ) + else: + from openai import AsyncOpenAI + client = AsyncOpenAI(**openai_params) + res = await client.chat.completions.create(messages=history, **payloads).model_dump() + else: + import openai + history, payloads = _build_request(model, params) - res = await openai.ChatCompletion.acreate(messages=history, **payloads) + res = await openai.ChatCompletion.acreate(messages=history, **payloads) text = "" async for r in res: From 1b615aedd280153314a4dd9187a8bdf991e87334 Mon Sep 17 00:00:00 2001 From: luchun <71970539+zhanghy-sketchzh@users.noreply.github.com> Date: Thu, 16 Nov 2023 22:39:18 +0800 Subject: [PATCH 2/3] Update chatgpt.py --- pilot/model/proxy/llms/chatgpt.py | 67 +++++++++++++++++-------------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/pilot/model/proxy/llms/chatgpt.py b/pilot/model/proxy/llms/chatgpt.py index 37bafe193..77354eb54 100644 --- a/pilot/model/proxy/llms/chatgpt.py +++ b/pilot/model/proxy/llms/chatgpt.py @@ -56,15 +56,16 @@ def _initialize_openai(params: ProxyModelParameters): return openai_params + def _initialize_openai_v1(params: ProxyModelParameters): try: from openai import OpenAI except ImportError as exc: - raise ValueError( - "Could not import python package: openai " - "Please install openai by command `pip install openai" - ) - + raise ValueError( + "Could not import python package: openai " + "Please install openai by command `pip install openai" + ) + api_type = params.proxy_api_type or os.getenv("OPENAI_API_TYPE", "open_ai") base_url = params.proxy_api_base or os.getenv( @@ -80,7 +81,7 @@ def _initialize_openai_v1(params: ProxyModelParameters): if not base_url and params.proxy_server_url: # Adapt previous proxy_server_url configuration base_url = params.proxy_server_url.split("/chat/completions")[0] - + if params.http_proxy: openai.proxies = params.http_proxy openai_params = { @@ -89,11 +90,7 @@ def _initialize_openai_v1(params: ProxyModelParameters): "proxies": params.http_proxy, } - return openai_params,api_type,api_version - - - - + return openai_params, api_type, api_version def _build_request(model: ProxyModel, params): @@ -101,7 +98,6 @@ def _build_request(model: ProxyModel, params): model_params = model.get_params() logger.info(f"Model: {model}, model_params: {model_params}") - messages: List[ModelMessage] = params["messages"] # Add history conversation @@ -134,10 +130,10 @@ def _build_request(model: ProxyModel, params): proxyllm_backend = model_params.proxyllm_backend if metadata.version("openai") >= "1.0.0": - openai_params,api_type,api_version = _initialize_openai_v1(model_params) + openai_params, api_type, api_version = _initialize_openai_v1(model_params) proxyllm_backend = proxyllm_backend or "gpt-3.5-turbo" payloads["model"] = proxyllm_backend - else: + else: openai_params = _initialize_openai(model_params) if openai_params["api_type"] == "azure": # engine = "deployment_name". @@ -158,20 +154,24 @@ def chatgpt_generate_stream( ): if metadata.version("openai") >= "1.0.0": model_params = model.get_params() - openai_params,api_type,api_version = _initialize_openai_v1(model_params) + openai_params, api_type, api_version = _initialize_openai_v1(model_params) history, payloads = _build_request(model, params) if api_type == "azure": from openai import AzureOpenAI + client = AzureOpenAI( - api_key = openai_params["api_key"], - api_version = api_version, - azure_endpoint = openai_params["base_url"] # Your Azure OpenAI resource's endpoint value. - ) + api_key=openai_params["api_key"], + api_version=api_version, + azure_endpoint=openai_params[ + "base_url" + ], # Your Azure OpenAI resource's endpoint value. + ) else: from openai import OpenAI + client = OpenAI(**openai_params) - print("openai_params",openai_params) - print("payloads",payloads) + print("openai_params", openai_params) + print("payloads", payloads) res = client.chat.completions.create(messages=history, **payloads) print(res) text = "" @@ -182,14 +182,14 @@ def chatgpt_generate_stream( yield text else: - import openai + history, payloads = _build_request(model, params) res = openai.ChatCompletion.create(messages=history, **payloads) text = "" - print("res",res) + print("res", res) for r in res: if r["choices"][0]["delta"].get("content") is not None: content = r["choices"][0]["delta"]["content"] @@ -202,22 +202,29 @@ async def async_chatgpt_generate_stream( ): if metadata.version("openai") >= "1.0.0": model_params = model.get_params() - openai_params,api_type,api_version = _initialize_openai_v1(model_params) + openai_params, api_type, api_version = _initialize_openai_v1(model_params) history, payloads = _build_request(model, params) if api_type == "azure": from openai import AsyncAzureOpenAI + client = AsyncAzureOpenAI( - api_key = openai_params["api_key"], - end_point = openai_params["base_url"], - api_version = api_version, - azure_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT") # Your Azure OpenAI resource's endpoint value. - ) + api_key=openai_params["api_key"], + end_point=openai_params["base_url"], + api_version=api_version, + azure_endpoint=os.getenv( + "AZURE_OPENAI_ENDPOINT" + ), # Your Azure OpenAI resource's endpoint value. + ) else: from openai import AsyncOpenAI + client = AsyncOpenAI(**openai_params) - res = await client.chat.completions.create(messages=history, **payloads).model_dump() + res = await client.chat.completions.create( + messages=history, **payloads + ).model_dump() else: import openai + history, payloads = _build_request(model, params) res = await openai.ChatCompletion.acreate(messages=history, **payloads) From 2ff9625d3fe5736bd228acee0c46ebe6ab72c4e5 Mon Sep 17 00:00:00 2001 From: luchun <71970539+zhanghy-sketchzh@users.noreply.github.com> Date: Thu, 16 Nov 2023 22:50:03 +0800 Subject: [PATCH 3/3] Update chatgpt.py --- pilot/model/proxy/llms/chatgpt.py | 33 ++++++++++++++++--------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/pilot/model/proxy/llms/chatgpt.py b/pilot/model/proxy/llms/chatgpt.py index 77354eb54..21d598d5a 100644 --- a/pilot/model/proxy/llms/chatgpt.py +++ b/pilot/model/proxy/llms/chatgpt.py @@ -170,10 +170,7 @@ def chatgpt_generate_stream( from openai import OpenAI client = OpenAI(**openai_params) - print("openai_params", openai_params) - print("payloads", payloads) res = client.chat.completions.create(messages=history, **payloads) - print(res) text = "" for r in res: if r.choices[0].delta.content is not None: @@ -209,19 +206,23 @@ async def async_chatgpt_generate_stream( client = AsyncAzureOpenAI( api_key=openai_params["api_key"], - end_point=openai_params["base_url"], api_version=api_version, - azure_endpoint=os.getenv( - "AZURE_OPENAI_ENDPOINT" - ), # Your Azure OpenAI resource's endpoint value. + azure_endpoint=openai_params[ + "base_url" + ], # Your Azure OpenAI resource's endpoint value. ) else: from openai import AsyncOpenAI client = AsyncOpenAI(**openai_params) - res = await client.chat.completions.create( - messages=history, **payloads - ).model_dump() + + res = await client.chat.completions.create(messages=history, **payloads) + text = "" + for r in res: + if r.choices[0].delta.content is not None: + content = r.choices[0].delta.content + text += content + yield text else: import openai @@ -229,9 +230,9 @@ async def async_chatgpt_generate_stream( res = await openai.ChatCompletion.acreate(messages=history, **payloads) - text = "" - async for r in res: - if r["choices"][0]["delta"].get("content") is not None: - content = r["choices"][0]["delta"]["content"] - text += content - yield text + text = "" + async for r in res: + if r["choices"][0]["delta"].get("content") is not None: + content = r["choices"][0]["delta"]["content"] + text += content + yield text