From 188afa7929d01032b72a5e71afa7070d0012b77e Mon Sep 17 00:00:00 2001 From: yhjun1026 <460342015@qq.com> Date: Thu, 14 Sep 2023 20:39:44 +0800 Subject: [PATCH 1/3] fix(ChatExcel): ChatExcel OutParse Bug Fix 1.ChatExcel OutParse Bug Fix --- pilot/out_parser/base.py | 86 +++++++------ .../chat_excel/excel_analyze/out_parser.py | 22 ++-- .../chat_excel/excel_learning/out_parser.py | 2 +- pilot/server/static/404.html | 2 +- pilot/server/static/404/index.html | 2 +- .../QgSLvD5Yu8RZmuZsIFsfR/_buildManifest.js | 1 + .../_ssgManifest.js | 0 .../static/chunks/161.0385a2869c97f681.js | 10 ++ .../static/chunks/161.a5c14a073f98c4a9.js | 10 -- .../static/chunks/196-48d8c6ab3e99146c.js | 39 ------ .../static/chunks/215-c0761f73cb28fff6.js | 1 - .../static/chunks/215-ed2d98cfbd44ae81.js | 1 + .../static/chunks/231.2702779548b5145a.js | 1 + .../static/chunks/34-52d4d2d11bef48dc.js | 16 +++ .../static/chunks/34-c924e44753dd2c5b.js | 16 --- .../static/chunks/378-dbd26a0c14558f18.js | 30 +++++ .../static/chunks/392.c534846271f9372d.js | 1 - .../static/chunks/455-597a8a48388a0d9b.js | 30 ----- .../static/chunks/542-f4cda9df864aa7ed.js | 19 +++ ...126d6ac9745.js => 566-cb742dc279bbfe42.js} | 0 .../static/chunks/582.74864353a0297601.js | 1 - .../static/chunks/582.92c95e34ff8f1f66.js | 1 + ...b79f1535087.js => 673-6b91681955aa1094.js} | 2 +- ...ae95bca674e.js => 718-8b4a2d7a281bb0c4.js} | 8 +- .../static/chunks/892-c40dbe9ae037747a.js | 40 ++++++ .../static/chunks/902-4b12ce531524546f.js | 19 --- .../static/chunks/908-33709e71961cb7a1.js | 3 - .../static/chunks/908-d76aabcc43706d37.js | 3 + .../chunks/pages/_app-3550d2087fc40b84.js | 114 ----------------- .../chunks/pages/_app-ba67ae83c876de13.js | 120 ++++++++++++++++++ ...f63f5aacee.js => chat-03c8b0d90000e1eb.js} | 2 +- ...52da89.js => database-e3f640bde30c17d7.js} | 2 +- ...c7f9.js => datastores-82628b0273c874d4.js} | 2 +- ...f44f2.js => documents-dea98f7e09e362f9.js} | 2 +- ...1447d.js => chunklist-87676458d42e378f.js} | 2 +- ...8efe06ace.js => index-704f93ece0dc096f.js} | 2 +- .../static/chunks/webpack-7691586e46740135.js | 1 + .../static/chunks/webpack-cc954be5f40cb903.js | 1 - .../_next/static/css/2054341936fbf0c0.css | 3 + .../_next/static/css/c8c143f6c7e4ba86.css | 3 - .../khlgfjIxxIMna2Uozo7tx/_buildManifest.js | 1 - pilot/server/static/chat/index.html | 2 +- pilot/server/static/database/index.html | 2 +- .../datastores/documents/chunklist/index.html | 2 +- .../static/datastores/documents/index.html | 2 +- pilot/server/static/datastores/index.html | 2 +- pilot/server/static/index.html | 2 +- pilot/server/static/models/baichuan.png | Bin 553 -> 0 bytes pilot/server/static/models/bard.gif | Bin 277497 -> 0 bytes pilot/server/static/models/chatglm.png | Bin 177042 -> 0 bytes pilot/server/static/models/chatgpt.png | Bin 4149 -> 0 bytes pilot/server/static/models/claude.png | Bin 391 -> 0 bytes pilot/server/static/models/falcon.jpeg | Bin 2732 -> 0 bytes pilot/server/static/models/google.png | Bin 3492 -> 0 bytes pilot/server/static/models/gorilla.png | Bin 9688 -> 0 bytes pilot/server/static/models/huggingface.svg | 37 ------ pilot/server/static/models/llama.jpg | Bin 2102 -> 0 bytes pilot/server/static/models/qwen2.png | Bin 5223 -> 0 bytes pilot/server/static/models/salesforce.jpeg | Bin 3546 -> 0 bytes pilot/server/static/models/vicuna.jpeg | Bin 1872 -> 0 bytes pilot/server/static/models/wizardlm.png | Bin 11812 -> 0 bytes pilot/server/static/models/zhipu.png | Bin 341 -> 0 bytes 62 files changed, 328 insertions(+), 342 deletions(-) create mode 100644 pilot/server/static/_next/static/QgSLvD5Yu8RZmuZsIFsfR/_buildManifest.js rename pilot/server/static/_next/static/{khlgfjIxxIMna2Uozo7tx => QgSLvD5Yu8RZmuZsIFsfR}/_ssgManifest.js (100%) create mode 100644 pilot/server/static/_next/static/chunks/161.0385a2869c97f681.js delete mode 100644 pilot/server/static/_next/static/chunks/161.a5c14a073f98c4a9.js delete mode 100644 pilot/server/static/_next/static/chunks/196-48d8c6ab3e99146c.js delete mode 100644 pilot/server/static/_next/static/chunks/215-c0761f73cb28fff6.js create mode 100644 pilot/server/static/_next/static/chunks/215-ed2d98cfbd44ae81.js create mode 100644 pilot/server/static/_next/static/chunks/231.2702779548b5145a.js create mode 100644 pilot/server/static/_next/static/chunks/34-52d4d2d11bef48dc.js delete mode 100644 pilot/server/static/_next/static/chunks/34-c924e44753dd2c5b.js create mode 100644 pilot/server/static/_next/static/chunks/378-dbd26a0c14558f18.js delete mode 100644 pilot/server/static/_next/static/chunks/392.c534846271f9372d.js delete mode 100644 pilot/server/static/_next/static/chunks/455-597a8a48388a0d9b.js create mode 100644 pilot/server/static/_next/static/chunks/542-f4cda9df864aa7ed.js rename pilot/server/static/_next/static/chunks/{566-493c6126d6ac9745.js => 566-cb742dc279bbfe42.js} (100%) delete mode 100644 pilot/server/static/_next/static/chunks/582.74864353a0297601.js create mode 100644 pilot/server/static/_next/static/chunks/582.92c95e34ff8f1f66.js rename pilot/server/static/_next/static/chunks/{673-9f0ceb79f1535087.js => 673-6b91681955aa1094.js} (77%) rename pilot/server/static/_next/static/chunks/{718-ae767ae95bca674e.js => 718-8b4a2d7a281bb0c4.js} (80%) create mode 100644 pilot/server/static/_next/static/chunks/892-c40dbe9ae037747a.js delete mode 100644 pilot/server/static/_next/static/chunks/902-4b12ce531524546f.js delete mode 100644 pilot/server/static/_next/static/chunks/908-33709e71961cb7a1.js create mode 100644 pilot/server/static/_next/static/chunks/908-d76aabcc43706d37.js delete mode 100644 pilot/server/static/_next/static/chunks/pages/_app-3550d2087fc40b84.js create mode 100644 pilot/server/static/_next/static/chunks/pages/_app-ba67ae83c876de13.js rename pilot/server/static/_next/static/chunks/pages/{chat-cf7d95f63f5aacee.js => chat-03c8b0d90000e1eb.js} (95%) rename pilot/server/static/_next/static/chunks/pages/{database-670b4fa76152da89.js => database-e3f640bde30c17d7.js} (97%) rename pilot/server/static/_next/static/chunks/pages/{datastores-56b24d3cabfcc7f9.js => datastores-82628b0273c874d4.js} (98%) rename pilot/server/static/_next/static/chunks/pages/datastores/{documents-e33cb051168f44f2.js => documents-dea98f7e09e362f9.js} (98%) rename pilot/server/static/_next/static/chunks/pages/datastores/documents/{chunklist-232ce3aa6271447d.js => chunklist-87676458d42e378f.js} (96%) rename pilot/server/static/_next/static/chunks/pages/{index-ad9fe4b8efe06ace.js => index-704f93ece0dc096f.js} (60%) create mode 100644 pilot/server/static/_next/static/chunks/webpack-7691586e46740135.js delete mode 100644 pilot/server/static/_next/static/chunks/webpack-cc954be5f40cb903.js create mode 100644 pilot/server/static/_next/static/css/2054341936fbf0c0.css delete mode 100644 pilot/server/static/_next/static/css/c8c143f6c7e4ba86.css delete mode 100644 pilot/server/static/_next/static/khlgfjIxxIMna2Uozo7tx/_buildManifest.js delete mode 100644 pilot/server/static/models/baichuan.png delete mode 100644 pilot/server/static/models/bard.gif delete mode 100644 pilot/server/static/models/chatglm.png delete mode 100644 pilot/server/static/models/chatgpt.png delete mode 100644 pilot/server/static/models/claude.png delete mode 100644 pilot/server/static/models/falcon.jpeg delete mode 100644 pilot/server/static/models/google.png delete mode 100644 pilot/server/static/models/gorilla.png delete mode 100644 pilot/server/static/models/huggingface.svg delete mode 100644 pilot/server/static/models/llama.jpg delete mode 100644 pilot/server/static/models/qwen2.png delete mode 100644 pilot/server/static/models/salesforce.jpeg delete mode 100644 pilot/server/static/models/vicuna.jpeg delete mode 100644 pilot/server/static/models/wizardlm.png delete mode 100644 pilot/server/static/models/zhipu.png diff --git a/pilot/out_parser/base.py b/pilot/out_parser/base.py index 2d080c3fe..ed6ed1b0d 100644 --- a/pilot/out_parser/base.py +++ b/pilot/out_parser/base.py @@ -129,44 +129,56 @@ def __illegal_json_ends(self, s): return temp_json def __extract_json(self, s): - temp_json = self.__json_interception(s, True) - if not temp_json: - temp_json = self.__json_interception(s) try: + # Get the dual-mode analysis first and get the maximum result + temp_json_simple = self.__json_interception(s) + temp_json_array = self.__json_interception(s, True) + if len(temp_json_simple) > len(temp_json_array): + temp_json = temp_json_simple + else: + temp_json = temp_json_array + + if not temp_json: + temp_json = self.__json_interception(s) + + temp_json = self.__illegal_json_ends(temp_json) return temp_json except Exception as e: - raise ValueError("Failed to find a valid json responseļ¼" + temp_json) + raise ValueError("Failed to find a valid json in LLM responseļ¼" + temp_json) def __json_interception(self, s, is_json_array: bool = False): - if is_json_array: - i = s.find("[") - if i < 0: - return None - count = 1 - for j, c in enumerate(s[i + 1 :], start=i + 1): - if c == "]": - count -= 1 - elif c == "[": - count += 1 - if count == 0: - break - assert count == 0 - return s[i : j + 1] - else: - i = s.find("{") - if i < 0: - return None - count = 1 - for j, c in enumerate(s[i + 1 :], start=i + 1): - if c == "}": - count -= 1 - elif c == "{": - count += 1 - if count == 0: - break - assert count == 0 - return s[i : j + 1] + try: + if is_json_array: + i = s.find("[") + if i < 0: + return "" + count = 1 + for j, c in enumerate(s[i + 1:], start=i + 1): + if c == "]": + count -= 1 + elif c == "[": + count += 1 + if count == 0: + break + assert count == 0 + return s[i: j + 1] + else: + i = s.find("{") + if i < 0: + return "" + count = 1 + for j, c in enumerate(s[i + 1:], start=i + 1): + if c == "}": + count -= 1 + elif c == "{": + count += 1 + if count == 0: + break + assert count == 0 + return s[i: j + 1] + except Exception as e: + return "" def parse_prompt_response(self, model_out_text) -> T: """ @@ -183,9 +195,9 @@ def parse_prompt_response(self, model_out_text) -> T: # if "```" in cleaned_output: # cleaned_output, _ = cleaned_output.split("```") if cleaned_output.startswith("```json"): - cleaned_output = cleaned_output[len("```json") :] + cleaned_output = cleaned_output[len("```json"):] if cleaned_output.startswith("```"): - cleaned_output = cleaned_output[len("```") :] + cleaned_output = cleaned_output[len("```"):] if cleaned_output.endswith("```"): cleaned_output = cleaned_output[: -len("```")] cleaned_output = cleaned_output.strip() @@ -194,9 +206,9 @@ def parse_prompt_response(self, model_out_text) -> T: cleaned_output = self.__extract_json(cleaned_output) cleaned_output = ( cleaned_output.strip() - .replace("\\n", " ") - .replace("\n", " ") - .replace("\\", " ") + .replace("\\n", " ") + .replace("\n", " ") + .replace("\\", " ") ) cleaned_output = self.__illegal_json_ends(cleaned_output) return cleaned_output diff --git a/pilot/scene/chat_data/chat_excel/excel_analyze/out_parser.py b/pilot/scene/chat_data/chat_excel/excel_analyze/out_parser.py index 3e5dfa68a..8bdbadadc 100644 --- a/pilot/scene/chat_data/chat_excel/excel_analyze/out_parser.py +++ b/pilot/scene/chat_data/chat_excel/excel_analyze/out_parser.py @@ -27,15 +27,19 @@ def __init__(self, sep: str, is_stream_out: bool): def parse_prompt_response(self, model_out_text): clean_str = super().parse_prompt_response(model_out_text) print("clean prompt response:", clean_str) - response = json.loads(clean_str) - for key in sorted(response): - if key.strip() == "sql": - sql = response[key] - if key.strip() == "thoughts": - thoughts = response[key] - if key.strip() == "display": - display = response[key] - return ExcelAnalyzeResponse(sql, thoughts, display) + try: + response = json.loads(clean_str) + for key in sorted(response): + if key.strip() == "sql": + sql = response[key] + if key.strip() == "thoughts": + thoughts = response[key] + if key.strip() == "display": + display = response[key] + return ExcelAnalyzeResponse(sql, thoughts, display) + except Exception as e: + raise ValueError(f"LLM Response Can't Parser! \n{ model_out_text}" ) + def parse_view_response(self, speak, data) -> str: ### tool out data to table view diff --git a/pilot/scene/chat_data/chat_excel/excel_learning/out_parser.py b/pilot/scene/chat_data/chat_excel/excel_learning/out_parser.py index b4dece331..0d636c16d 100644 --- a/pilot/scene/chat_data/chat_excel/excel_learning/out_parser.py +++ b/pilot/scene/chat_data/chat_excel/excel_learning/out_parser.py @@ -41,7 +41,7 @@ def parse_prompt_response(self, model_out_text): return model_out_text def parse_view_response(self, speak, data) -> str: - if data: + if data and not isinstance(data, str): ### tool out data to table view html_title = f"### **Data Summary**\n{data.desciption} " html_colunms = f"### **Data Structure**\n" diff --git a/pilot/server/static/404.html b/pilot/server/static/404.html index 966f5bb82..cda6129d2 100644 --- a/pilot/server/static/404.html +++ b/pilot/server/static/404.html @@ -1 +1 @@ -