From fcf9fd75cb19a091adf949821a12e3e6a9ddd957 Mon Sep 17 00:00:00 2001 From: Amo-Zeng <51903518+Amo-Zeng@users.noreply.github.com> Date: Mon, 6 May 2024 09:48:34 +0000 Subject: [PATCH 1/5] =?UTF-8?q?=E6=A8=A1=E4=BB=BF=E4=B8=B4=E6=97=B6key?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=B8=B4=E6=97=B6chatgpt=E6=A0=BC=E5=BC=8Fap?= =?UTF-8?q?i=E6=8E=A5=E5=85=A5=E6=96=B9=E6=B3=95=E3=80=82=E5=9C=A8?= =?UTF-8?q?=E8=BE=93=E5=85=A5=E6=A1=86=E8=BE=93=E5=85=A5=E5=A6=82=E4=B8=8B?= =?UTF-8?q?json=E5=8F=AF=E4=BB=A5=E4=B8=B4=E6=97=B6=E5=88=87=E6=8D=A2?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E8=AF=A5api=EF=BC=8C=E9=A1=B5=E9=9D=A2?= =?UTF-8?q?=E5=88=B7=E6=96=B0=E5=8D=B3=E5=A4=B1=E6=95=88=E3=80=82{"api=5Fk?= =?UTF-8?q?ey":"sk-heK1pbXG1JzHk84oBd03B2C1A1504587Ba8cEd40CbD2Bxxf","tmp?= =?UTF-8?q?=5Fendpoint":"http://xxxx/v1/chat/completions","tmp=5Fmodel":"g?= =?UTF-8?q?pt-3.5-turbo"}?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request_llms/bridge_chatgpt.py | 20 ++++++++++++++++---- toolbox.py | 6 ++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/request_llms/bridge_chatgpt.py b/request_llms/bridge_chatgpt.py index 2e4ebe3e4..6569a99ae 100644 --- a/request_llms/bridge_chatgpt.py +++ b/request_llms/bridge_chatgpt.py @@ -91,7 +91,7 @@ def predict_no_ui_long_connection(inputs:str, llm_kwargs:dict, history:list=[], try: # make a POST request to the API endpoint, stream=False from .bridge_all import model_info - endpoint = verify_endpoint(model_info[llm_kwargs['llm_model']]['endpoint']) + endpoint = verify_endpoint(model_info[llm_kwargs['llm_model']]['endpoint']) if not llm_kwargs['tmp_endpoint'] else llm_kwargs['tmp_endpoint'] response = requests.post(endpoint, headers=headers, proxies=proxies, json=payload, stream=True, timeout=TIMEOUT_SECONDS); break except requests.exceptions.ReadTimeout as e: @@ -147,7 +147,12 @@ def predict_no_ui_long_connection(inputs:str, llm_kwargs:dict, history:list=[], raise ConnectionAbortedError("正常结束，但显示Token不足，导致输出不完整，请削减单次输入的文本量。") return result - +def is_any_tmp_model(inputs): + try: + tmp_model_info=json.loads(inputs).keys() + return "api_key" in tmp_model_info and "tmp_model" in tmp_model_info and "tmp_model" in tmp_model_info + except: + return False def predict(inputs:str, llm_kwargs:dict, plugin_kwargs:dict, chatbot:ChatBotWithCookies, history:list=[], system_prompt:str='', stream:bool=True, additional_fn:str=None): """ @@ -164,6 +169,13 @@ def predict(inputs:str, llm_kwargs:dict, plugin_kwargs:dict, chatbot:ChatBotWith chatbot.append(("输入已识别为openai的api_key", what_keys(inputs))) yield from update_ui(chatbot=chatbot, history=history, msg="api_key已导入") # 刷新界面 return + elif is_any_tmp_model(inputs): + chatbot._cookies['api_key'] = json.loads(inputs)['api_key'] + chatbot._cookies['tmp_model'] = json.loads(inputs)['tmp_model'] + chatbot._cookies['tmp_endpoint'] = json.loads(inputs)['tmp_endpoint'] + chatbot.append(("输入已识别为临时openai格式的模型，页面刷新后将失效", '临时模型：'+json.loads(inputs)['tmp_model'])) + yield from update_ui(chatbot=chatbot, history=history, msg="临时模型已导入") # 刷新界面 + return elif not is_any_api_key(chatbot._cookies['api_key']): chatbot.append((inputs, "缺少api_key。\n\n1. 临时解决方案：直接在输入区键入api_key，然后回车提交。\n\n2. 长效解决方案：在config.py中配置。")) yield from update_ui(chatbot=chatbot, history=history, msg="缺少api_key") # 刷新界面 @@ -195,7 +207,7 @@ def predict(inputs:str, llm_kwargs:dict, plugin_kwargs:dict, chatbot:ChatBotWith # 检查endpoint是否合法 try: from .bridge_all import model_info - endpoint = verify_endpoint(model_info[llm_kwargs['llm_model']]['endpoint']) + endpoint = verify_endpoint(model_info[llm_kwargs['llm_model']]['endpoint']) if not llm_kwargs['tmp_endpoint'] else llm_kwargs['tmp_endpoint'] except: tb_str = '```\n' + trimmed_format_exc() + '```' chatbot[-1] = (inputs, tb_str) @@ -383,7 +395,7 @@ def generate_payload(inputs, llm_kwargs, history, system_prompt, stream): logging.info("Random select model:" + model) payload = { - "model": model, + "model": model if not llm_kwargs['tmp_model'] else llm_kwargs['tmp_model'] , "messages": messages, "temperature": llm_kwargs['temperature'], # 1.0, "top_p": llm_kwargs['top_p'], # 1.0, diff --git a/toolbox.py b/toolbox.py index e11efc126..ceb6199b9 100644 --- a/toolbox.py +++ b/toolbox.py @@ -101,12 +101,16 @@ def decorated(request: gradio.Request, cookies:dict, max_length:int, llm_model:s cookies.update({ 'top_p': top_p, 'api_key': cookies['api_key'], + 'tmp_model': cookies['tmp_model'], + 'tmp_endpoint': cookies['tmp_endpoint'], 'llm_model': llm_model, 'temperature': temperature, 'user_name': user_name, }) llm_kwargs = { 'api_key': cookies['api_key'], + 'tmp_model': cookies['tmp_model'], + 'tmp_endpoint': cookies['tmp_endpoint'], 'llm_model': llm_model, 'top_p': top_p, 'max_length': max_length, @@ -607,6 +611,8 @@ def load_chat_cookies(): "api_key": API_KEY, "llm_model": LLM_MODEL, "customize_fn_overwrite": customize_fn_overwrite_, + "tmp_model":'', + "tmp_endpoint":'', } From 77588ee0718810f4f01ec77143ba3cd8d48e5888 Mon Sep 17 00:00:00 2001 From: Amo-Zeng <51903518+Amo-Zeng@users.noreply.github.com> Date: Sat, 11 May 2024 02:17:43 +0000 Subject: [PATCH 2/5] =?UTF-8?q?=E5=A2=9E=E5=8A=A0tmp=5Fkey=E4=BB=A5?= =?UTF-8?q?=E5=85=BC=E5=AE=B9=E4=B8=80=E4=BA=9B=E4=B8=8D=E6=98=AF=E6=A0=87?= =?UTF-8?q?=E5=87=86=E6=A0=BC=E5=BC=8F=E7=9A=84key?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request_llms/bridge_chatgpt.py | 6 +++--- toolbox.py | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/request_llms/bridge_chatgpt.py b/request_llms/bridge_chatgpt.py index 6569a99ae..6c3a6bd80 100644 --- a/request_llms/bridge_chatgpt.py +++ b/request_llms/bridge_chatgpt.py @@ -150,7 +150,7 @@ def predict_no_ui_long_connection(inputs:str, llm_kwargs:dict, history:list=[], def is_any_tmp_model(inputs): try: tmp_model_info=json.loads(inputs).keys() - return "api_key" in tmp_model_info and "tmp_model" in tmp_model_info and "tmp_model" in tmp_model_info + return "tmp_key" in tmp_model_info and "tmp_model" in tmp_model_info and "tmp_endpoint" in tmp_model_info except: return False def predict(inputs:str, llm_kwargs:dict, plugin_kwargs:dict, chatbot:ChatBotWithCookies, @@ -170,7 +170,7 @@ def predict(inputs:str, llm_kwargs:dict, plugin_kwargs:dict, chatbot:ChatBotWith yield from update_ui(chatbot=chatbot, history=history, msg="api_key已导入") # 刷新界面 return elif is_any_tmp_model(inputs): - chatbot._cookies['api_key'] = json.loads(inputs)['api_key'] + chatbot._cookies['tmp_key'] = json.loads(inputs)['tmp_key'] chatbot._cookies['tmp_model'] = json.loads(inputs)['tmp_model'] chatbot._cookies['tmp_endpoint'] = json.loads(inputs)['tmp_endpoint'] chatbot.append(("输入已识别为临时openai格式的模型，页面刷新后将失效", '临时模型：'+json.loads(inputs)['tmp_model'])) @@ -342,7 +342,7 @@ def generate_payload(inputs, llm_kwargs, history, system_prompt, stream): headers = { "Content-Type": "application/json", - "Authorization": f"Bearer {api_key}" + "Authorization": f"Bearer {api_key if not llm_kwargs['tmp_key'] else llm_kwargs['tmp_key'] }" } if API_ORG.startswith('org-'): headers.update({"OpenAI-Organization": API_ORG}) if llm_kwargs['llm_model'].startswith('azure-'): diff --git a/toolbox.py b/toolbox.py index ceb6199b9..be921d4e6 100644 --- a/toolbox.py +++ b/toolbox.py @@ -101,6 +101,7 @@ def decorated(request: gradio.Request, cookies:dict, max_length:int, llm_model:s cookies.update({ 'top_p': top_p, 'api_key': cookies['api_key'], + 'tmp_key': cookies['tmp_key'], 'tmp_model': cookies['tmp_model'], 'tmp_endpoint': cookies['tmp_endpoint'], 'llm_model': llm_model, @@ -109,6 +110,7 @@ def decorated(request: gradio.Request, cookies:dict, max_length:int, llm_model:s }) llm_kwargs = { 'api_key': cookies['api_key'], + 'tmp_key': cookies['tmp_key'], 'tmp_model': cookies['tmp_model'], 'tmp_endpoint': cookies['tmp_endpoint'], 'llm_model': llm_model, @@ -611,6 +613,7 @@ def load_chat_cookies(): "api_key": API_KEY, "llm_model": LLM_MODEL, "customize_fn_overwrite": customize_fn_overwrite_, + "tmp_key":'', "tmp_model":'', "tmp_endpoint":'', } From 315567f8e15c010662efc9dcaaf8ce51fba74491 Mon Sep 17 00:00:00 2001 From: Amo-Zeng <51903518+Amo-Zeng@users.noreply.github.com> Date: Sat, 11 May 2024 02:29:53 +0000 Subject: [PATCH 3/5] =?UTF-8?q?=E5=A2=9E=E5=8A=A0tmp=5Fkey=E4=BB=A5?= =?UTF-8?q?=E5=85=BC=E5=AE=B9=E4=B8=80=E4=BA=9B=E4=B8=8D=E6=98=AF=E6=A0=87?= =?UTF-8?q?=E5=87=86=E6=A0=BC=E5=BC=8F=E7=9A=84key=EF=BC=8C=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E4=B8=B4=E6=97=B6=E6=A8=A1=E5=9E=8B=E6=A0=BC=E5=BC=8F?= =?UTF-8?q?=E6=8F=90=E7=A4=BA=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request_llms/bridge_chatgpt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/request_llms/bridge_chatgpt.py b/request_llms/bridge_chatgpt.py index 6c3a6bd80..475dbc7bf 100644 --- a/request_llms/bridge_chatgpt.py +++ b/request_llms/bridge_chatgpt.py @@ -332,8 +332,8 @@ def generate_payload(inputs, llm_kwargs, history, system_prompt, stream): """ 整合所有信息，选择LLM模型，生成http请求，为发送请求做准备 """ - if not is_any_api_key(llm_kwargs['api_key']): - raise AssertionError("你提供了错误的API_KEY。\n\n1. 临时解决方案：直接在输入区键入api_key，然后回车提交。\n\n2. 长效解决方案：在config.py中配置。") + if not is_any_api_key(llm_kwargs['api_key']) and not llm_kwargs['tmp_key']: + raise AssertionError('你提供了错误的API_KEY。\n\n1. 临时解决方案：直接在输入区键入api_key，然后回车提交。\n\n2. 长效解决方案：在config.py中配置。\n\n3.接入临时模型：在输入区键入以下格式临时模型信息{"tmp_key":"xxx","tmp_endpoint":"https://xxxx.xxx","tmp_model":"gpt-3.5-turbo-16k"}，然后回车提交') if llm_kwargs['llm_model'].startswith('vllm-'): api_key = 'no-api-key' From 9a967093a5791903f17deb7cbec832c49fc5acf1 Mon Sep 17 00:00:00 2001 From: Amo-Zeng <51903518+Amo-Zeng@users.noreply.github.com> Date: Sat, 11 May 2024 02:40:00 +0000 Subject: [PATCH 4/5] =?UTF-8?q?=E5=A2=9E=E5=8A=A0tmp=5Fkey=E4=BB=A5?= =?UTF-8?q?=E5=85=BC=E5=AE=B9=E4=B8=80=E4=BA=9B=E4=B8=8D=E6=98=AF=E6=A0=87?= =?UTF-8?q?=E5=87=86=E6=A0=BC=E5=BC=8F=E7=9A=84key=EF=BC=8C=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E4=B8=B4=E6=97=B6=E6=A8=A1=E5=9E=8B=E6=A0=BC=E5=BC=8F?= =?UTF-8?q?=E6=8F=90=E7=A4=BA=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request_llms/bridge_chatgpt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/request_llms/bridge_chatgpt.py b/request_llms/bridge_chatgpt.py index 475dbc7bf..287d337ca 100644 --- a/request_llms/bridge_chatgpt.py +++ b/request_llms/bridge_chatgpt.py @@ -176,8 +176,8 @@ def predict(inputs:str, llm_kwargs:dict, plugin_kwargs:dict, chatbot:ChatBotWith chatbot.append(("输入已识别为临时openai格式的模型，页面刷新后将失效", '临时模型：'+json.loads(inputs)['tmp_model'])) yield from update_ui(chatbot=chatbot, history=history, msg="临时模型已导入") # 刷新界面 return - elif not is_any_api_key(chatbot._cookies['api_key']): - chatbot.append((inputs, "缺少api_key。\n\n1. 临时解决方案：直接在输入区键入api_key，然后回车提交。\n\n2. 长效解决方案：在config.py中配置。")) + elif not is_any_api_key(chatbot._cookies['api_key']) and not chatbot._cookies['tmp_key']: + chatbot.append((inputs, '缺少API_KEY。\n\n1. 临时解决方案：直接在输入区键入api_key，然后回车提交。\n\n2. 长效解决方案：在config.py中配置。\n\n3.接入临时模型：在输入区键入以下格式临时模型信息{"tmp_key":"xxx","tmp_endpoint":"https://xxxx.xxx","tmp_model":"gpt-3.5-turbo-16k"}，然后回车提交')) yield from update_ui(chatbot=chatbot, history=history, msg="缺少api_key") # 刷新界面 return From 5c7a381865d178b9760d31c7671359086135dba1 Mon Sep 17 00:00:00 2001 From: Amo-Zeng <51903518+Amo-Zeng@users.noreply.github.com> Date: Sat, 11 May 2024 02:47:49 +0000 Subject: [PATCH 5/5] =?UTF-8?q?=E5=A2=9E=E5=8A=A0tmp=5Fkey=E4=BB=A5?= =?UTF-8?q?=E5=85=BC=E5=AE=B9=E4=B8=80=E4=BA=9B=E4=B8=8D=E6=98=AF=E6=A0=87?= =?UTF-8?q?=E5=87=86=E6=A0=BC=E5=BC=8F=E7=9A=84key=EF=BC=8C=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E4=B8=B4=E6=97=B6=E6=A8=A1=E5=9E=8B=E6=A0=BC=E5=BC=8F?= =?UTF-8?q?=E6=8F=90=E7=A4=BA=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- request_llms/bridge_chatgpt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/request_llms/bridge_chatgpt.py b/request_llms/bridge_chatgpt.py index 287d337ca..858344954 100644 --- a/request_llms/bridge_chatgpt.py +++ b/request_llms/bridge_chatgpt.py @@ -337,12 +337,14 @@ def generate_payload(inputs, llm_kwargs, history, system_prompt, stream): if llm_kwargs['llm_model'].startswith('vllm-'): api_key = 'no-api-key' + elif llm_kwargs['tmp_key']: + api_key = llm_kwargs['tmp_key'] else: api_key = select_api_key(llm_kwargs['api_key'], llm_kwargs['llm_model']) headers = { "Content-Type": "application/json", - "Authorization": f"Bearer {api_key if not llm_kwargs['tmp_key'] else llm_kwargs['tmp_key'] }" + "Authorization": f"Bearer {api_key}" } if API_ORG.startswith('org-'): headers.update({"OpenAI-Organization": API_ORG}) if llm_kwargs['llm_model'].startswith('azure-'):